/** * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, Cell k) throws IOException { int result = s.seekTo(k); if (result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if (result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
static boolean reseekAtOrAfter(HFileScanner s, Cell k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); }
private boolean walkForwardInSingleRow(final HFileScanner scanner, final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) throws IOException { boolean foundCandidate = false; do { Cell kv = scanner.getKeyValue(); // If we are not in the row, skip. if (this.comparator.compareRows(kv, firstOnRow) < 0) continue; // Did we go beyond the target row? If so break. if (state.isTooFar(kv, firstOnRow)) break; if (state.isExpired(kv)) { continue; } // If we added something, this row is a contender. break. if (state.handle(kv)) { foundCandidate = true; break; } } while (scanner.next()); return foundCandidate; }
@Override public byte[] getLastKey() { if (top) { return super.getLastKey(); } // Get a scanner that caches the block and that uses pread. HFileScanner scanner = getScanner(true, true); try { if (scanner.seekBefore(this.splitkey)) { return Bytes.toBytes(scanner.getKey()); } } catch (IOException e) { LOG.warn("Failed seekBefore " + Bytes.toStringBinary(this.splitkey), e); } return null; }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); // System.out.println("Random row: " + new String(b)); Cell c = createCell(b); if (scanner.seekTo(c) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } // TODO: HFileScanner doesn't do Cells yet. Temporary fix. c = scanner.getKeyValue(); // System.out.println("Found row: " + // new String(c.getRowArray(), c.getRowOffset(), c.getRowLength())); PerformanceEvaluationCommons.assertKey(b, c); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } c = scanner.getKeyValue(); PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (StoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = scanner.getKeyValue().getRow(); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }
public HFileSortedIterator(HFileScanner scan, SerializedComparator comparator, byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive) throws IOException { this.scan = scan; this.comparator = comparator; this.from = from; this.fromInclusive = fromInclusive; this.to = to; this.toInclusive = toInclusive; assert from == null || to == null || comparator.compare(from, 0, from.length, to, 0, to.length) <= 0; start = sopConfig.getStatistics().getScan().begin(); foundNext = evalFrom(); }
public HFileSortedIterator(HFileScanner scan, byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive) throws IOException { this.scan = scan; this.from = from; this.fromInclusive = fromInclusive; this.to = to; this.toInclusive = toInclusive; scanStat = (stats == null) ? new SortedOplogStatistics("", "").new ScanOperation( 0, 0, 0, 0, 0, 0, 0) : stats.getScan(); scanStart = scanStat.begin(); if (scan == null) { return; } assert from == null || to == null || scan.getReader().getComparator().compare(from, to) <= 0; initIterator(); }
static boolean reseekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { // This function is similar to seekAtOrAfter function int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); if (result <= 0) { // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } else { // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); } }
private boolean walkForwardInSingleRow(final HFileScanner scanner, final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) throws IOException { boolean foundCandidate = false; do { KeyValue kv = scanner.getKeyValue(); // If we are not in the row, skip. if (this.comparator.compareRows(kv, firstOnRow) < 0) continue; // Did we go beyond the target row? If so break. if (state.isTooFar(kv, firstOnRow)) break; if (state.isExpired(kv)) { continue; } // If we added something, this row is a contender. break. if (state.handle(kv)) { foundCandidate = true; break; } } while (scanner.next()); return foundCandidate; }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); if (scanner.seekTo(b) != 0) { System.out.println("Nonexistent row: " + new String(b)); return; } ByteBuffer k = scanner.getKey(); PerformanceEvaluationCommons.assertKey(b, k); // System.out.println("Found row: " + new String(b)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { System.out.println("NOTHING FOLLOWS"); } ByteBuffer v = scanner.getValue(); PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); } }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (StoreFile f : this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte[] row = scanner.getKeyValue().getRow(); if (Bytes.equals(row, STARTROW)) { count1++; } else if (Bytes.equals(row, secondRowBytes)) { count2++; } } while (scanner.next()); } assertEquals(countRow1, count1); assertEquals(countRow2, count2); }
private void readHFileSeq(Path file, Compression.Algorithm compression) throws Exception { CacheConfig cacheConf = new CacheConfig(conf); HFile.Reader reader = HFile.createReader(hdfs, file, cacheConf, conf); HFileScanner scanner = reader.getScanner(true, true, false); scanner.seekTo(); @SuppressWarnings("unused") KeyValue kv = null; while (scanner.next()) { kv = scanner.getKeyValue(); //logger.debug("key: {} value: {}", new String (kv.getKey()), new String (kv.getValue())); } }
private void readHFileSeqId(Path file, Compression.Algorithm compression) throws Exception { CacheConfig cacheConf = new CacheConfig(conf); HFile.Reader reader = HFile.createReader(hdfs, file, cacheConf, conf); HFileScanner scanner = reader.getScanner(true, true, false); @SuppressWarnings("unused") KeyValue kv = null; scanner.seekTo(); for (int i = 0; i < testSize; i++) { scanner.seekTo(getKey(i).getBytes()); kv = scanner.getKeyValue(); //logger.debug("key: {} value: {}", new String (kv.getKey()), new String (kv.getValue())); } }
private void readHFileRandom(Path file, Compression.Algorithm compression) throws Exception { CacheConfig cacheConf = new CacheConfig(conf); HFile.Reader reader = HFile.createReader(hdfs, file, cacheConf, conf); HFileScanner scanner = reader.getScanner(true, true, false); @SuppressWarnings("unused") KeyValue kv = null; scanner.seekTo(); Random random = new Random(); for (int i = 0; i < testSize; i++) { scanner.seekTo(); scanner.seekTo(getKey(random.nextInt(testSize)).getBytes()); kv = scanner.getKeyValue(); //logger.debug("key: {} value: {}", new String (kv.getKey()), new String (kv.getValue())); } }
/** * Issues an HFile lookup on the underlying HFile.Reader. This is protected * for testing. */ protected Pair<ByteBuffer, Pair<ByteBuffer, Throwable>> getValueFromHFile(ByteBuffer key) { try { HFileScanner scanner = reader.getScanner(true, true, false); KeyValue kv = buildKeyValueForLookup( BytesUtil.readBytesFromByteBufferWithoutConsume(key)); int code = scanner.seekTo(kv.getKey()); ByteBuffer value = null; if (code == 0) { value = ByteBuffer.wrap(scanner.getKeyValue().getValue()); if (this.sizeStatsKey != null) { Stats.addMetric(this.sizeStatsKey, value.remaining()); } Stats.addMetric("value-size", value.remaining()); } else { Stats.incr("not-found"); if (this.notFoundStatsKey != null) { Stats.incr(this.notFoundStatsKey); } } return new ImmutablePair(key, new ImmutablePair(value, null)); } catch (Throwable t) { return new ImmutablePair(key, new ImmutablePair(null, t)); } }
/** * * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, Cell k) throws IOException { int result = s.seekTo(k); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if(result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (StoreFile f: this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = scanner.getKeyValue().getRow(); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }
/** * * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { int result = s.seekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if(result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
static boolean reseekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); }
private boolean walkForwardInSingleRow(final HFileScanner scanner, final KeyValue firstOnRow, final GetClosestRowBeforeTracker state) throws IOException { boolean foundCandidate = false; do { KeyValue kv = scanner.getKeyValue(); // If we are not in the row, skip. if (this.comparator.compareRows(kv, firstOnRow) < 0) continue; // Did we go beyond the target row? If so break. if (state.isTooFar(kv, firstOnRow)) break; if (state.isExpired(kv)) { continue; } // If we added something, this row is a contender. break. if (state.handle(kv)) { foundCandidate = true; break; } } while(scanner.next()); return foundCandidate; }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); if (scanner.seekTo(b) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } ByteBuffer k = scanner.getKey(); PerformanceEvaluationCommons.assertKey(b, k); // System.out.println("Found row: " + new String(b)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } ByteBuffer v = scanner.getValue(); PerformanceEvaluationCommons.assertValueSize(v.limit(), ROW_LENGTH); } }
private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf, Configuration conf) throws IOException { final IndexHalfStoreFileReader halfreader = new IndexHalfStoreFileReader(fs, p, cacheConf, bottom, conf); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); KeyValue getseekTorowKey3 = getSeekToRowKey(); scanner.seekTo(getseekTorowKey3.getBuffer(), 8, 17); boolean next = scanner.next(); KeyValue keyValue = null; if (next) { keyValue = scanner.getKeyValue(); } byte[] expectedRow = getExpected(); byte[] actualRow = keyValue.getRow(); Assert.assertArrayEquals(expectedRow, actualRow); halfreader.close(true); }
static boolean reseekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); if (result <= 0) { // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } else { // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); } }
@Override public Optional<Cell> getLastKey() { if (top) { return super.getLastKey(); } // Get a scanner that caches the block and that uses pread. HFileScanner scanner = getScanner(true, true); try { if (scanner.seekBefore(this.splitCell)) { return Optional.ofNullable(scanner.getKey()); } } catch (IOException e) { LOG.warn("Failed seekBefore " + Bytes.toStringBinary(this.splitkey), e); } finally { if (scanner != null) { scanner.close(); } } return Optional.empty(); }
