@Override public Iterator<String[]> iterator() throws Exception{ Iterator<CSVRecord> iterCSVRecords = this.getCSVParser().iterator(); Iterator<String[]> iterStringArrays = Iterators.transform(iterCSVRecords, (CSVRecord input) -> { Iterator<String> iterCols = input.iterator(); List<String> cols = new ArrayList(); while(iterCols.hasNext()){ cols.add(iterCols.next()); } String[] output = cols.toArray(new String[0]); return output; }); return iterStringArrays; }
@Override public void write(String outputFilePath) throws Exception{ try(Writer out = new BufferedWriter(new FileWriter(outputFilePath)); CSVPrinter csvPrinter = new CSVPrinter(out, CSVFormat.RFC4180)) { if(this.getHeaders() != null){ csvPrinter.printRecord(this.getHeaders()); } Iterator<CSVRecord> recordIter = this.getCSVParser().iterator(); while(recordIter.hasNext()){ CSVRecord record = recordIter.next(); csvPrinter.printRecord(record); } csvPrinter.flush(); }catch(Exception e){ throw e; } }
@Override public void accept(CSVRecord theRecord) { String code = theRecord.get("LOINC_NUM"); if (isNotBlank(code)) { String longCommonName = theRecord.get("LONG_COMMON_NAME"); String shortName = theRecord.get("SHORTNAME"); String consumerName = theRecord.get("CONSUMER_NAME"); String display = firstNonBlank(longCommonName, shortName, consumerName); ConceptEntity concept = new ConceptEntity(myCodeSystemVersion, code); concept.setDisplay(display); Validate.isTrue(!myCode2Concept.containsKey(code)); myCode2Concept.put(code, concept); } }
private ArrayList<Map<String, String>> parse() { final ArrayList<Map<String, String>> list; final CSVParser parser; final URL resource; final CSVFormat csvFormat; final Charset charset; list = new ArrayList<Map<String, String>> (); try { resource = ResourceHelper.getResource(isFromGameResourceInput(),fileName); charset= FileEncodingDetectorHelper.guessEncodingAndGetCharset(resource); csvFormat = CSVFormatHelper.getDefaultCSVFormat(); parser = CSVParser.parse(resource, charset, csvFormat); for (CSVRecord record : parser ) list.add((Map<String, String>)record.toMap()); } catch (IOException e) { log.error("Fail", e); } return list; }
private ArrayList<Map<String,String>> parse() { ArrayList<Map<String,String>> list; CSVParser parser; URL resource; CSVFormat csvFormat; Charset charset; list = new ArrayList<Map<String,String>> (); try { resource = ResourceHelper.getResource(isFromGameResourceInput(),fileName); charset= FileEncodingDetectorHelper.guessEncodingAndGetCharset(resource); csvFormat = CSVFormatHelper.getDefaultCSVFormat(); parser = CSVParser.parse(resource, charset, csvFormat); for (CSVRecord record : parser ) list.add(record.toMap()); } catch (IOException e) { log.error("Fail", e); } return list; }
private List <Tag> createTags(CSVParser parser, Course course) { tagMap = new HashMap<>(); ArrayList<Tag> tagList = new ArrayList<Tag>(); for (CSVRecord csvRecord : parser) { String name = csvRecord.get(TagPoolColumn.TAG.ordinal()).toString(); if (!tagMap.containsKey(name)){ Tag tag = new Tag(); tag.setName(name); tag.setCourse(course); tagMap.put(name, tag); tag.setNumber(tagMap.size()); tagList.add(tag); } } return tagList; }
public List<Map<String,String>> parse() throws BuenOjoCSVParserException { List<Map<String,String>> list = new ArrayList<>(); CSVParser parser = null; try { parser = CSVFormat.RFC4180.withHeader() .withDelimiter(',') .withAllowMissingColumnNames(true) .parse(new InputStreamReader(this.inputStreamSource.getInputStream())); } catch (IOException e) { throw new BuenOjoCSVParserException(e.getMessage()); } for (CSVRecord record :parser) { Map<String,String> map = record.toMap(); list.add(map); } return list; }
public List<PhotoLocationSightPair> parse () throws IOException, BuenOjoCSVParserException { CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream())); List<CSVRecord> records = parser.getRecords(); if (records.size() == 0 ) { throw new BuenOjoCSVParserException("El archivos de miras no contiene registros"); } ArrayList<PhotoLocationSightPair> sightPairs = new ArrayList<>(records.size()); for (CSVRecord record : records) { PhotoLocationSightPair sight = new PhotoLocationSightPair(); sight.setNumber(new Integer(record.get(PhotoLocationSightPairCSVColumn.id))); sight.setSatelliteX(new Integer(record.get(PhotoLocationSightPairCSVColumn.satCol))); sight.setSatelliteY(new Integer(record.get(PhotoLocationSightPairCSVColumn.satRow))); sight.setSatelliteTolerance(new Integer(record.get(PhotoLocationSightPairCSVColumn.satTolerancia))); sight.setTerrainX(new Integer(record.get(PhotoLocationSightPairCSVColumn.terCol))); sight.setTerrainY(new Integer(record.get(PhotoLocationSightPairCSVColumn.terRow))); sight.setTerrainTolerance(new Integer(record.get(PhotoLocationSightPairCSVColumn.terTolerancia))); sightPairs.add(sight); } return sightPairs; }
public List<TagPair> parse() throws IOException { CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream())); ArrayList<TagPair> tagPairs = new ArrayList<>(AVG_ITEMS); for (CSVRecord record : parser ){ TagPair pair = new TagPair(); Integer tagSlotId = new Integer(record.get("id")); Integer tagNumber = new Integer(record.get("etiqueta")); pair.setTagSlotId(tagSlotId); Optional<Tag> optionalTag = tagList.stream().filter(isEqualToTagNumber(tagNumber)).findFirst(); if (optionalTag.isPresent()){ Tag tag = optionalTag.get(); pair.setTag(tag); tagPairs.add(pair); }else { log.debug("Attempt to get invalid tag with number: "+tagNumber); } } return tagPairs; }
public List<TagCircle> parse() throws IOException, BuenOjoCSVParserException { ArrayList<TagCircle> list = new ArrayList<>(MAX_CIRCLES); CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(false).parse(new InputStreamReader(this.inputStream)); for (CSVRecord record : parser ){ TagCircle circle = new TagCircle(); circle.setNumber(new Integer(record.get("id"))); circle.setX(new Integer(record.get("col"))); circle.setY(new Integer(record.get("row"))); circle.setRadioPx(new Float(record.get("radioPx"))); list.add(circle); } if (list.size()>MAX_CIRCLES){ throw new BuenOjoCSVParserException("el archivo contiene mas de "+MAX_CIRCLES+ "áreas circulares"); } return list; }
public Integer[] parseLevels() throws BuenOjoCSVParserException, IOException{ CSVRecord record = getRecords().get(0); String levels = record.get(PhotoLocationLandscapeLevelsCSVColumns.levels.ordinal()); //Matcher m = Pattern.compile("(\\d*):(\\d*)").matcher(levels); String[] m = levels.split(":"); if (m==null) { throw new BuenOjoCSVParserException("el formato del nivel es incorrecto: "+levels); } Integer[] n = new Integer[PhotoLocationLevelComponents.count.ordinal()]; int lower = PhotoLocationLevelComponents.lowerLevel.ordinal(); int higher = PhotoLocationLevelComponents.higherLevel.ordinal(); n[lower] = new Integer(m[lower]); n[higher] = new Integer(m[higher]); return n; }
/** * Creates a new dataset with column labels and data read from the given Reader, using a specified input format. * * @param reader the Reader to read column labels and data from * @param input_format the format */ @SuppressWarnings("WeakerAccess") public DataSet(final Reader reader, final CSVFormat input_format) { this(); try (CSVParser parser = new CSVParser(reader, input_format.withHeader())) { labels.addAll(getColumnLabels(parser)); for (final CSVRecord record : parser) { final List<String> items = csvRecordToList(record); final int size = items.size(); // Don't add row if the line was empty. if (size > 1 || (size == 1 && items.get(0).length() > 0)) { records.add(items); } } reader.close(); } catch (final IOException e) { throw new RuntimeException(e); } }
/** * This method reads the file from @see {@link ReadFolder} and put into an array list the data we need. * We use here the API commons-csv. * Attention : tu run with the API you need to import him into the project @see README. * @param folderName. * @exception IOException | NumberFormatException : print error reading file. */ public void read(String folderName) { try { Reader in = new FileReader(folderName + "/" + file.getName()); BufferedReader br = new BufferedReader(in); String firstLine = br.readLine(); if (checkTheFile(firstLine)) { Iterable<CSVRecord> records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(br); for (CSVRecord record : records) if (record.get("Type").equals("WIFI") && !record.get("FirstSeen").contains("1970")) inputWifi(record, firstLine); in.close(); br.close(); } } catch(IOException | NumberFormatException ex) { // If there is an error. System.out.println("Error reading file : " + ex); System.exit(0); } }
@Override public Iterator<Row> iterator() { return new Iterator<BulkUserImportReader.Row>() { Iterator<CSVRecord> i = csvParser.iterator(); @Override public boolean hasNext() { return i.hasNext(); } @Override public Row next() { CSVRecord record = i.next(); return new Row(record); } @Override public void remove() { i.remove(); } }; }
private void compareCSVFiles(String actualPath, String expectedPath) { try (CSVParser parserTranslatedFile = createCsvParser(actualPath, ",")) { try (CSVParser parserExpectedFile = createCsvParser(expectedPath, ",")) { Iterator<CSVRecord> translatedIterator = parserTranslatedFile.iterator(); Iterator<CSVRecord> expectedIterator = parserExpectedFile.iterator(); while (translatedIterator.hasNext() && expectedIterator.hasNext()) { CSVRecord translatedRecord = translatedIterator.next(); CSVRecord expectedRecord = expectedIterator.next(); compareCSVRecords(translatedRecord, expectedRecord); } if (translatedIterator.hasNext()) { fail("Actual file is longer (has more lines) than expected file"); } if (expectedIterator.hasNext()) { fail("Expected file is longer (has more lines) than actual file"); } } } catch (Exception e) { fail("Exception while iterating over files"); } }
/** * sauvegarde dans la base de données * @param parser * @return */ public int updateDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String serie = item.get(0).trim(); String mention = item.get(1).trim(); String specialite = item.get(2).trim(); String section = item.get(3).trim(); //enregistrer if (add(serie, mention, specialite, section)){ res++; } } DBManager.quit(); return res; }
/** * sauvegarde dans la base de données * @param parser * @return */ public int updateDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String id = item.get(0).trim(); String serie = item.get(1).trim(); String mention = item.get(2).trim(); String specialite = item.get(3).trim(); String section = item.get(4).trim(); //enregistrer if (add(id, serie, mention, specialite, section)){ res++; } } DBManager.quit(); return res; }
/** * sauvegarde dans la base de données * @param parser * @return */ public int updateDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String id = item.get(0).trim(); String code = item.get(1).trim(); String note = item.get(2).trim(); if (note.isEmpty()) { note=null; } //enregistrer if (add(id, code, note)){ res++; } } DBManager.quit(); return res; }
/** * sauvegarde dans la base de données des foreign keys * @param parser * @return */ public int updateFkDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String id = item.get(0).trim(); String code = item.get(1).trim(); String matiere = item.get(3).trim(); //enregistrer if (!matiere.isEmpty()){ if (addMatiere(id, code, matiere)){ res++; } } } DBManager.quit(); return res; }
/** * sauvegarde dans la base de données des foreign keys * @param parser * @return */ public int updateFkDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String code = item.get(0).trim(); String composition = item.get(2).trim(); String rattrapage = item.get(3).trim(); //enregistrer if (!composition.isEmpty()){ if (addComposition(code, composition)){ res++; } } if (!rattrapage.isEmpty()){ if (addRattrapage(code, rattrapage)){ res++; } } } DBManager.quit(); return res; }
/** * sauvegarde dans la base de données * @param parser * @return */ public int updateDB(CSVParser parser) { int res = 0; DBManager.connect(); for (CSVRecord item : parser) { String code = item.get(0).trim(); String libelle = item.get(1).trim(); String epreuve = item.get(2).trim(); //enregistrer if (add(code, libelle, epreuve)){ res++; } } DBManager.quit(); return res; }
@Override public void accept(CSVRecord theRecord) { String id = theRecord.get("id"); String date = theRecord.get("effectiveTime"); if (!myConceptIdToMostRecentDate.containsKey(id) || myConceptIdToMostRecentDate.get(id).compareTo(date) < 0) { boolean active = "1".equals(theRecord.get("active")); if (active) { myValidConceptIds.add(id); } else { myValidConceptIds.remove(id); } myConceptIdToMostRecentDate.put(id, date); } }
@Override public String next() { CSVRecord record = inner.next(); StringWriter json = new StringWriter(); try { JsonGenerator gen = jsonFactory.createJsonGenerator(json); gen.writeStartObject(); for (CSVHeaderMap.Entry entry : headerMap.entries()) { String name = entry.getName(); String value = record.get(entry.getIndex()); gen.writeFieldName(name); entry.getWriter().write(gen, value); } gen.writeEndObject(); gen.close(); } catch (IOException e) { throw new RuntimeException(e); } return json.toString(); }
@Test public void testFindSpreadsheetsFromCells() throws IOException { CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"), Charset.forName("utf-8"), CSVFormat.DEFAULT); List<Cell> cells = new ArrayList<>(); for (CSVRecord record : parse) { cells.add(new Cell(Float.parseFloat(record.get(0)), Float.parseFloat(record.get(1)), Float.parseFloat(record.get(2)), Float.parseFloat(record.get(3)))); } List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES); Collections.sort(expected, Rectangle.ILL_DEFINED_ORDER); List<Rectangle> foundRectangles = SpreadsheetExtractionAlgorithm.findSpreadsheetsFromCells(cells); Collections.sort(foundRectangles, Rectangle.ILL_DEFINED_ORDER); assertTrue(foundRectangles.equals(expected)); }
@Test public void testNaturalOrderOfRectanglesOneMoreTime() throws IOException { CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestBasicExtractor-RECTANGLE_TEST_NATURAL_ORDER.csv"), Charset.forName("utf-8"), CSVFormat.DEFAULT); List<Rectangle> rectangles = new ArrayList<>(); for (CSVRecord record : parse) { rectangles.add(new Rectangle(Float.parseFloat(record.get(0)), Float.parseFloat(record.get(1)), Float.parseFloat(record.get(2)), Float.parseFloat(record.get(3)))); } //List<Rectangle> rectangles = Arrays.asList(RECTANGLES_TEST_NATURAL_ORDER); Utils.sort(rectangles, Rectangle.ILL_DEFINED_ORDER); for (int i = 0; i < (rectangles.size() - 1); i++) { Rectangle rectangle = rectangles.get(i); Rectangle nextRectangle = rectangles.get(i + 1); assertTrue(rectangle.compareTo(nextRectangle) < 0); } }
public HdrData read(String filename) throws IOException { Reader in = new FileReader(filename); HdrData ret = new HdrData(); List<Double> value = ret.getValue(); List<Double> percentile = ret.getPercentile(); Iterable<CSVRecord> records = CSVFormat.RFC4180 .withCommentMarker('#') .withFirstRecordAsHeader() .parse(in); for (CSVRecord record : records) { String valueStr = record.get(0); String percentileStr = record.get(1); logger.debug("Value: {}", valueStr); logger.debug("Percentile: {}", percentileStr); value.add(Double.parseDouble(valueStr)); percentile.add(Double.parseDouble(percentileStr) * 100); } return ret; }
@Override public void accept(CSVRecord theRecord) { for (IBaseResource resource : resources) { if (resource instanceof AllergyIntolerance) { AllergyIntolerance allergy = (AllergyIntolerance) resource; if (allergy.getIdentifier().size()>0 && theRecord.get("allergy.identifier").equals(allergy.getIdentifier().get(0).getValue())) { AllergyIntolerance.AllergyIntoleranceReactionComponent reaction = allergy.addReaction(); reaction.addManifestation().addCoding() .setSystem(CareConnectSystem.SNOMEDCT) .setDisplay(theRecord.get("reaction.manifestation.display")) .setCode(theRecord.get("reaction.manifestation")); // System.out.println(ctx.newJsonParser().setPrettyPrint(true).encodeResourceToString(allergy)); } } } }
@ProcessElement public void processElement(ProcessContext c) { String rawInput = null; InputContent iContent = null; try { rawInput = c.element(); if (rawInput == null) throw new Exception("ParseCSVFile: null raw content"); FileIndexerPipelineOptions options = c.getPipelineOptions().as(FileIndexerPipelineOptions.class); Integer textColumnIdx = options.getTextColumnIdx(); Integer collectionItemIdIdx = options.getCollectionItemIdIdx(); InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(rawInput,StandardCharsets.UTF_8.name())); Iterable<CSVRecord> records = CSVFormat.DEFAULT .withFirstRecordAsHeader() .parse(isr); for (CSVRecord record : records) { String text = record.get(textColumnIdx); String documentCollectionId = IndexerPipelineUtils.DOC_COL_ID_CSV_FILE; String collectionItemId = record.get(collectionItemIdIdx); InputContent ic = new InputContent( null /*url*/, null /*pubTime*/, null /*title*/, null /*author*/, null /*language*/, text, documentCollectionId, collectionItemId, 0 /*skipIndexing*/); c.output(ic); } } catch (Exception e) { LOG.warn(e.getMessage()); } }
@ProcessElement public void processElement(ProcessContext c) { String rawInput = null; InputContent iContent = null; try { rawInput = c.element(); if (rawInput == null) throw new Exception("ParseCSVLine: null raw content"); rawInput = rawInput.trim(); if (rawInput.isEmpty()) throw new Exception("ParseCSVLine: empty raw content or whitespace chars only"); FileIndexerPipelineOptions options = c.getPipelineOptions().as(FileIndexerPipelineOptions.class); Integer textColumnIdx = options.getTextColumnIdx(); Integer collectionItemIdIdx = options.getCollectionItemIdIdx(); InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(rawInput,StandardCharsets.UTF_8.name())); Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(isr); for (CSVRecord record : records) { // should only be one record, but handle multi-record case as well String text = record.get(textColumnIdx); String documentCollectionId = IndexerPipelineUtils.DOC_COL_ID_CSV_FILE; String collectionItemId = record.get(collectionItemIdIdx); InputContent ic = new InputContent( null /*url*/, null /*pubTime*/, null /*title*/, null /*author*/, null /*language*/, text, documentCollectionId, collectionItemId, 0 /*skipIndexing*/); c.output(ic); } } catch (Exception e) { LOG.warn(e.getMessage()); } }
private CSVRecord getNextRecord() { CSVRecord record; try { record = this.iterator.next(); } catch (NoSuchElementException var2) { record = null; } return record; }
@Override public void accept(CSVRecord theRecord) { String parentCode = theRecord.get("IMMEDIATE_PARENT"); String childCode = theRecord.get("CODE"); String childCodeText = theRecord.get("CODE_TEXT"); if (isNotBlank(parentCode) && isNotBlank(childCode)) { ConceptEntity parent = getOrCreate(parentCode, "(unknown)"); ConceptEntity child = getOrCreate(childCode, childCodeText); parent.addChild(child, ConceptParentChildLink.RelationshipTypeEnum.ISA); } }
public void parseAndInject(PhotoLocationExercise exercise, InputStreamSource inputStreamSource) throws BuenOjoCSVParserException, IOException{ CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(inputStreamSource.getInputStream())); List<CSVRecord> records = parser.getRecords(); if (records.size() > 1) { throw new BuenOjoCSVParserException("El archivo contiene más de un ejercicio"); } if (records.size() == 0) { throw new BuenOjoCSVParserException("El archivo de ejericio es inválido"); } CSVRecord record = records.get(0); String name = record.get(MetadataColumns.name); String description = record.get(MetadataColumns.description); String difficulty = record.get(MetadataColumns.difficulty); String seconds = record.get(MetadataColumns.seconds); String totalScore = record.get(MetadataColumns.totalScore.ordinal()); String imageName = record.get(MetadataColumns.imageName.ordinal()); exercise.setDescription(description); exercise.setName(name); exercise.setDifficulty(difficultyFromString(difficulty)); exercise.setTotalTimeInSeconds(new Integer(seconds)); exercise.setTotalScore(new Float(totalScore)); exercise.setExtraPhotosCount(3); List<PhotoLocationImage> imgs = photoLocationImageRepository.findAll(); Optional<PhotoLocationImage> opt = imgs.stream().filter(p -> p.getImage().getName().equals(imageName)).collect(Collectors.toList()).stream().findFirst(); if (!opt.isPresent()){ throw new BuenOjoCSVParserException("la imagen '"+imageName+"' no existe en la base de datos"); } PhotoLocationImage img = opt.get(); img.setKeywords(exercise.getLandscapeKeywords()); photoLocationImageRepository.save(img); exercise.setTerrainPhoto(img); }
private List<TagPool> createTagPool(CSVParser parser) throws BuenOjoCSVParserException{ ArrayList<TagPool> list = new ArrayList<>(); for (CSVRecord record : parser) { String name = record.get(TagPoolColumn.TAG.ordinal()).toString(); Tag tag = tagMap.get(name); for (int i = TagPoolColumn.SIMILAR_1.ordinal(); (i <record.size()) && i <= TagPoolColumn.SIMILAR_3.ordinal(); i++) { String similarTagName = record.get(i); if (similarTagName != null && !StringUtils.isAnyEmpty(similarTagName)){ Tag similarTag = tagMap.get(similarTagName); if (similarTag != null){ TagPool tagPool = new TagPool(); tagPool.setTag(tag); tagPool.setSimilarTag(similarTag); tagPool.setSimilarity(i); list.add(tagPool); } else { throw new BuenOjoCSVParserException("no se pudo obtener la etiqueta con nombre: '"+similarTagName+"'"); } } } } return list; }
private List<CSVRecord> getRecords () throws BuenOjoCSVParserException, IOException{ CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream())); List<CSVRecord> records = parser.getRecords(); if (records.size()==0) { throw new BuenOjoCSVParserException("El archivo no contiene palabras clave"); } if (records.size()>1) { throw new BuenOjoCSVParserException("El archivo contiene mas de un registro de palabras clave"); } return records; }
private PhotoLocationKeyword parseRecord(CSVRecord record){ PhotoLocationKeyword keyword = new PhotoLocationKeyword(); String description = record.get(PhotoLocationKeywordColumns.description.ordinal()); keyword.setDescription(description); String name = record.get(PhotoLocationKeywordColumns.name.ordinal()); keyword.setName(name); return keyword; }
private static List<String> csvRecordToList(final CSVRecord record) { final List<String> list = new ArrayList<>(); for (final String value : record) { list.add(value); } return list; }
private void startWorkers(Iterator<CSVRecord> iter, long targetRecordCount, WorkerPool workers) throws Exception { while (iter.hasNext()) { long currentRecord = 0; List<Map<String, String>> sub = new ArrayList<Map<String, String>>(); while (iter.hasNext() && currentRecord < targetRecordCount) { sub.add(iter.next().toMap()); currentRecord++; } Constructor<Worker> constructor = workerClass.getConstructor(Iterator.class, Map.class, JanusGraph.class); Worker worker = constructor.newInstance(sub.iterator(), propertiesMap, graph); workers.submit(worker); } //main thread would wait here workers.wait4Finish(); }
public void loadFile(String fileName, Map<String, Object> propertiesMap, WorkerPool workers) throws Exception { log.info("Loading " + fileName); // long linesCount = BatchHelper.countLines(fileName); this.propertiesMap = propertiesMap; Reader in = new FileReader(fileName); Iterator<CSVRecord> iter = CSVFormat.EXCEL.withHeader().parse(in).iterator(); // long freeMemory = Runtime.getRuntime().freeMemory()/1024/1024; // TODO Calculate targetThreadCount using the free memory and number of threads // to execute // Max record count per thread startWorkers(iter, Config.getConfig().getWorkersTargetRecordCount(), workers); }
private void startWorkers(Iterator<CSVRecord> iter, long targetRecordCount, WorkerPool workers) throws Exception { while (iter.hasNext()) { long currentRecord = 0; List<Map<String,String>> sub = new ArrayList<Map<String,String>>(); while (iter.hasNext() && currentRecord < targetRecordCount) { sub.add(iter.next().toMap()); currentRecord++; } Constructor<Worker> constructor = workerClass.getConstructor(Iterator.class, Map.class, JanusGraph.class); Worker worker = constructor.newInstance(sub.iterator(), propertiesMap, graph); workers.submit(worker); } }