/** * Compare families, qualifiers, and timestamps of the given Cells. * They are assumed to be of the same row. * Nulls are after non-nulls. */ private static int compareCellKeysWithinRow(Cell c1, Cell c2) { if (c1 == null) { return 1; // source missing cell } if (c2 == null) { return -1; // target missing cell } int result = CellComparator.compareFamilies(c1, c2); if (result != 0) { return result; } result = CellComparator.compareQualifiers(c1, c2); if (result != 0) { return result; } // note timestamp comparison is inverted - more recent cells first return CellComparator.compareTimestamps(c1, c2); }
/** Clean up the current data block */ private void finishBlock() throws IOException { if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return; // Update the first data block offset for scanning. if (firstDataBlockOffset == -1) { firstDataBlockOffset = outputStream.getPos(); } // Update the last data block offset lastDataBlockOffset = outputStream.getPos(); fsBlockWriter.writeHeaderAndData(outputStream); int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader(); Cell indexEntry = CellComparator.getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock); dataBlockIndexWriter.addEntry(CellUtil.getCellKeySerializedAsKeyValueKey(indexEntry), lastDataBlockOffset, onDiskSize); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (cacheConf.shouldCacheDataOnWrite()) { doCacheOnWrite(lastDataBlockOffset); } }
@Test public void testScanForwards() throws IOException { CellSearcher searcher = null; try { searcher = DecoderFactory.checkOut(block, true); int i = -1; while (searcher.advance()) { ++i; KeyValue inputCell = rows.getInputs().get(i); Cell outputCell = searcher.current(); // check all 3 permutations of equals() Assert.assertEquals(inputCell, outputCell); Assert.assertEquals(outputCell, inputCell); Assert.assertTrue(CellComparator.equals(inputCell, outputCell)); } Assert.assertEquals(rows.getInputs().size(), i + 1); } finally { DecoderFactory.checkIn(searcher); } }
@Override public void individualSearcherAssertions(CellSearcher searcher) { assertRowOffsetsCorrect(); searcher.resetToBeforeFirstEntry(); //test first cell try { searcher.advance(); } catch (IOException e) { throw new RuntimeException(e); } Cell first = searcher.current(); Assert.assertTrue(CellComparator.equals(d.get(0), first)); //test first cell in second row Assert.assertTrue(searcher.positionAt(d.get(1))); Assert.assertTrue(CellComparator.equals(d.get(1), searcher.current())); testBetween1and2(searcher); testBetween2and3(searcher); }
private void testBetween1and2(CellSearcher searcher){ CellScannerPosition p;//reuse Cell betweenAAndAAA = new KeyValue(AA, cf, cq, ts-2, v); //test exact Assert.assertFalse(searcher.positionAt(betweenAAndAAA)); //test atOrBefore p = searcher.positionAtOrBefore(betweenAAndAAA); Assert.assertEquals(CellScannerPosition.BEFORE, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(1))); //test atOrAfter p = searcher.positionAtOrAfter(betweenAAndAAA); Assert.assertEquals(CellScannerPosition.AFTER, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(2))); }
private void testBetween2and3(CellSearcher searcher){ CellScannerPosition p;//reuse Cell betweenAAAndB = new KeyValue(AAA, cf, cq, ts-2, v); //test exact Assert.assertFalse(searcher.positionAt(betweenAAAndB)); //test atOrBefore p = searcher.positionAtOrBefore(betweenAAAndB); Assert.assertEquals(CellScannerPosition.BEFORE, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(2))); //test atOrAfter p = searcher.positionAtOrAfter(betweenAAAndB); Assert.assertEquals(CellScannerPosition.AFTER, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(3))); }
public void testBasicLoadValue() throws Exception { KeyValue [] kvs = genKVs(row, family, value, 1, 100); Arrays.sort(kvs, CellComparator.getInstance()); Result r = Result.create(kvs); ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024); for (int i = 0; i < 100; ++i) { final byte[] qf = Bytes.toBytes(i); loadValueBuffer.clear(); r.loadValue(family, qf, loadValueBuffer); loadValueBuffer.flip(); assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i)))); assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), r.getValueAsByteBuffer(family, qf)); } }
/** * Binary search for keys in indexes. * * @param arr array of byte arrays to search for * @param key the key you want to find * @param comparator a comparator to compare. * @return zero-based index of the key, if the key is present in the array. * Otherwise, a value -(i + 1) such that the key is between arr[i - * 1] and arr[i] non-inclusively, where i is in [0, i], if we define * arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above * means that this function can return 2N + 1 different values * ranging from -(N + 1) to N - 1. * @return the index of the block */ public static int binarySearch(Cell[] arr, Cell key, CellComparator comparator) { int low = 0; int high = arr.length - 1; while (low <= high) { int mid = (low+high) >>> 1; // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. int cmp = comparator.compare(key, arr[mid]); // key lives above the midpoint if (cmp > 0) low = mid + 1; // key lives below the midpoint else if (cmp < 0) high = mid - 1; // BAM. how often does this really happen? else return mid; } return - (low+1); }
public void testMultiVersionLoadValue() throws Exception { KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length]; System.arraycopy(kvs1, 0, kvs, 0, kvs1.length); System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length); Arrays.sort(kvs, CellComparator.getInstance()); ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024); Result r = Result.create(kvs); for (int i = 0; i < 100; ++i) { final byte[] qf = Bytes.toBytes(i); loadValueBuffer.clear(); r.loadValue(family, qf, loadValueBuffer); loadValueBuffer.flip(); assertEquals(loadValueBuffer, ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i)))); assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), r.getValueAsByteBuffer(family, qf)); } }
public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy) throws IOException { super(conf, c); this.store = store; this.regionServices = regionServices; this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = createMemStoreCompactor(compactionPolicy); if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) { // if user requested to work with MSLABs (whether on- or off-heap), then the // immutable segments are going to use CellChunkMap as their index indexType = IndexType.CHUNK_MAP; } else { indexType = IndexType.ARRAY_MAP; } // initialization of the flush size should happen after initialization of the index type // so do not transfer the following method initInmemoryFlushSize(conf); }
/** * Ensure that expired delete family markers don't override valid puts */ @Test public void testExpiredDeleteFamily() throws Exception { long now = System.currentTimeMillis(); KeyValue[] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, KeyValue.Type.DeleteFamily), create("R1", "cf", "a", now-10, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.readVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false); try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) { List<Cell> results = new ArrayList<>(); assertEquals(true, scanner.next(results)); assertEquals(1, results.size()); assertEquals(kvs[1], results.get(0)); results.clear(); assertEquals(false, scanner.next(results)); } }
/** * Note maxVersion and minVersion must set according to cf's conf, not user's scan parameter. * * @param columns columns specified user in query * @param comparartor the cell comparator * @param minVersion The minimum number of versions to keep(used when TTL is set). * @param maxVersion The maximum number of versions in CF's conf * @param resultMaxVersions maximum versions to return per column, which may be different from * maxVersion * @param oldestUnexpiredTS the oldest timestamp we are interested in, based on TTL */ public NewVersionBehaviorTracker(NavigableSet<byte[]> columns, CellComparator comparartor, int minVersion, int maxVersion, int resultMaxVersions, long oldestUnexpiredTS) { this.maxVersions = maxVersion; this.minVersions = minVersion; this.resultMaxVersions = resultMaxVersions; this.oldestStamp = oldestUnexpiredTS; if (columns != null && columns.size() > 0) { this.columns = new byte[columns.size()][]; int i = 0; for (byte[] column : columns) { this.columns[i++] = column; } } this.comparator = comparartor; reset(); }
/** * @param targetBoundaries The boundaries on which writers/files are separated. * @param majorRangeFrom Major range is the range for which at least one file should be written * (because all files are included in compaction). majorRangeFrom is the left boundary. * @param majorRangeTo The right boundary of majorRange (see majorRangeFrom). */ public BoundaryMultiWriter(CellComparator comparator, List<byte[]> targetBoundaries, byte[] majorRangeFrom, byte[] majorRangeTo) throws IOException { super(comparator); this.boundaries = targetBoundaries; this.existingWriters = new ArrayList<>(this.boundaries.size() - 1); // "major" range (range for which all files are included) boundaries, if any, // must match some target boundaries, let's find them. assert (majorRangeFrom == null) == (majorRangeTo == null); if (majorRangeFrom != null) { majorRangeFromIndex = Arrays.equals(majorRangeFrom, StripeStoreFileManager.OPEN_KEY) ? 0 : Collections .binarySearch(boundaries, majorRangeFrom, Bytes.BYTES_COMPARATOR); majorRangeToIndex = Arrays.equals(majorRangeTo, StripeStoreFileManager.OPEN_KEY) ? boundaries.size() : Collections.binarySearch(boundaries, majorRangeTo, Bytes.BYTES_COMPARATOR); if (this.majorRangeFromIndex < 0 || this.majorRangeToIndex < 0) { throw new IOException("Major range does not match writer boundaries: [" + Bytes.toString(majorRangeFrom) + "] [" + Bytes.toString(majorRangeTo) + "]; from " + majorRangeFromIndex + " to " + majorRangeToIndex); } } }
public void testBasicGetColumn() throws Exception { KeyValue [] kvs = genKVs(row, family, value, 1, 100); Arrays.sort(kvs, CellComparator.getInstance()); Result r = Result.create(kvs); for (int i = 0; i < 100; ++i) { final byte[] qf = Bytes.toBytes(i); List<Cell> ks = r.getColumnCells(family, qf); assertEquals(1, ks.size()); assertTrue(CellUtil.matchingQualifier(ks.get(0), qf)); assertEquals(ks.get(0), r.getColumnLatestCell(family, qf)); } }
private ScanInfo(byte[] family, int minVersions, int maxVersions, long ttl, KeepDeletedCells keepDeletedCells, long timeToPurgeDeletes, CellComparator comparator, long tableMaxRowSize, boolean usePread, long cellsPerTimeoutCheck, boolean parallelSeekEnabled, long preadMaxBytes, boolean newVersionBehavior) { this.family = family; this.minVersions = minVersions; this.maxVersions = maxVersions; this.ttl = ttl; this.keepDeletedCells = keepDeletedCells; this.timeToPurgeDeletes = timeToPurgeDeletes; this.comparator = comparator; this.tableMaxRowSize = tableMaxRowSize; this.usePread = usePread; this.cellsPerTimeoutCheck = cellsPerTimeoutCheck; this.parallelSeekEnabled = parallelSeekEnabled; this.preadMaxBytes = preadMaxBytes; this.newVersionBehavior = newVersionBehavior; }
public void testMultiVersionGetColumn() throws Exception { KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length]; System.arraycopy(kvs1, 0, kvs, 0, kvs1.length); System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length); Arrays.sort(kvs, CellComparator.getInstance()); Result r = Result.create(kvs); for (int i = 0; i < 100; ++i) { final byte[] qf = Bytes.toBytes(i); List<Cell> ks = r.getColumnCells(family, qf); assertEquals(2, ks.size()); assertTrue(CellUtil.matchingQualifier(ks.get(0), qf)); assertEquals(200, ks.get(0).getTimestamp()); assertEquals(ks.get(0), r.getColumnLatestCell(family, qf)); } }
private void verifyScan(Scan s, long expectedRows, long expectedKeys) throws IOException { InternalScanner scanner = this.region.getScanner(s); List<Cell> results = new ArrayList<>(); int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.getInstance()); LOG.info("counter=" + i + ", " + results); if (results.isEmpty()) break; assertTrue("Scanned too many rows! Only expected " + expectedRows + " total but already scanned " + (i+1), expectedRows > i); assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + results.size(), expectedKeys, results.size()); results.clear(); } assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i); }
public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFileContext context = new HFileContextBuilder() .withCompression(AbstractHFileWriter.compressionByName(codec)).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withFileContext(context) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf); try { reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); // position to the start of file // Scanner does not do Cells yet. Do below for now till fixed. cc = scanner.getKeyValue(); if (CellComparator.compareRows(c, cc) != 0) { throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString()); } } finally { reader.close(); } }
@Test public void testThree() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1")); final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"), Bytes.toBytes("2")); final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"), Bytes.toBytes("3")); encoder.write(kv1); encoder.write(kv2); encoder.write(kv3); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current(); assertTrue(CellComparator.equals(c, kv1)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv2)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv3)); assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
/** * Compare two Cells considering reversed scanner. * ReversedScanner only reverses rows, not columns. */ private int compare(Cell a, Cell b) { int r = 0; if (currentRegion != null && currentRegion.isMetaRegion()) { r = metaComparator.compareRows(a, b); } else { r = CellComparator.compareRows(a, b); } if (r != 0) { return this.scan.isReversed() ? -r : r; } return CellComparator.compareWithoutRow(a, b); }
@Override public boolean equals(Object obj) { if (!(obj instanceof Cell)) { return false; } //Temporary hack to maintain backwards compatibility with KeyValue.equals return CellComparator.equalsIgnoreMvccVersion(this, (Cell)obj); //TODO return CellComparator.equals(this, (Cell)obj);//see HBASE-6907 }
@Test public void testRandomSeekHits() throws IOException { CellSearcher searcher = null; try { searcher = DecoderFactory.checkOut(block, true); for (KeyValue kv : rows.getInputs()) { boolean hit = searcher.positionAt(kv); Assert.assertTrue(hit); Cell foundKv = searcher.current(); Assert.assertTrue(CellComparator.equals(kv, foundKv)); } } finally { DecoderFactory.checkIn(searcher); } }
@Override public List<Integer> getRowStartIndexes() { List<Integer> rowStartIndexes = Lists.newArrayList(); rowStartIndexes.add(0); List<KeyValue> inputs = getInputs(); for (int i = 1; i < inputs.size(); ++i) { KeyValue lastKv = inputs.get(i - 1); KeyValue kv = inputs.get(i); if (!CellComparator.equalsRow(lastKv, kv)) { rowStartIndexes.add(i); } } return rowStartIndexes; }
@Test public void testThree() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1")); final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"), Bytes.toBytes("2")); final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"), Bytes.toBytes("3")); encoder.write(kv1); encoder.write(kv2); encoder.write(kv3); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current(); assertTrue(CellComparator.equals(c, kv1)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv2)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv3)); assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
