/*********************** construct ******************************/ // pass in blockMeta so we can initialize buffers big enough for all cells in the block public PrefixTreeArrayScanner(PrefixTreeBlockMeta blockMeta, int rowTreeDepth, int rowBufferLength, int qualifierBufferLength, int tagsBufferLength) { this.rowNodes = new RowNodeReader[rowTreeDepth]; for (int i = 0; i < rowNodes.length; ++i) { rowNodes[i] = new RowNodeReader(); } this.rowBuffer = new byte[rowBufferLength]; this.familyBuffer = new byte[PrefixTreeBlockMeta.MAX_FAMILY_LENGTH]; this.familyReader = new ColumnReader(familyBuffer, ColumnNodeType.FAMILY); this.qualifierBuffer = new byte[qualifierBufferLength]; this.tagsBuffer = new byte[tagsBufferLength]; this.qualifierReader = new ColumnReader(qualifierBuffer, ColumnNodeType.QUALIFIER); this.tagsReader = new ColumnReader(tagsBuffer, ColumnNodeType.TAGS); this.timestampDecoder = new TimestampDecoder(); this.mvccVersionDecoder = new MvccVersionDecoder(); }
@Before public void compile() throws IOException { // Always run with tags. But should also ensure that KVs without tags work fine os = new ByteArrayOutputStream(1 << 20); encoder = new PrefixTreeEncoder(os, includeMemstoreTS); inputKvs = rows.getInputs(); for (KeyValue kv : inputKvs) { encoder.write(kv); } encoder.flush(); totalBytes = encoder.getTotalBytes(); blockMetaWriter = encoder.getBlockMeta(); outputBytes = os.toByteArray(); // start reading, but save the assertions for @Test methods buffer = ByteBuffer.wrap(outputBytes); blockMetaReader = new PrefixTreeBlockMeta(buffer); searcher = new PrefixTreeArraySearcher(blockMetaReader, blockMetaReader.getRowTreeDepth(), blockMetaReader.getMaxRowLength(), blockMetaReader.getMaxQualifierLength(), blockMetaReader.getMaxTagsLength()); searcher.initOnBlock(blockMetaReader, outputBytes, includeMemstoreTS); }
public TestTimestampEncoder(TestTimestampData testTimestamps) throws IOException { this.timestamps = testTimestamps; this.blockMeta = new PrefixTreeBlockMeta(); this.blockMeta.setNumMetaBytes(0); this.blockMeta.setNumRowBytes(0); this.blockMeta.setNumQualifierBytes(0); this.encoder = new LongEncoder(); for (Long ts : testTimestamps.getInputs()) { encoder.add(ts); } encoder.compile(); blockMeta.setTimestampFields(encoder); bytes = encoder.getByteArray(); decoder = new TimestampDecoder(); decoder.initOnBlock(blockMeta, bytes); }
/*********************** construct ******************************/ // pass in blockMeta so we can initialize buffers big enough for all cells in the block public PrefixTreeArrayScanner(PrefixTreeBlockMeta blockMeta, int rowTreeDepth, int rowBufferLength, int qualifierBufferLength) { this.rowNodes = new RowNodeReader[rowTreeDepth]; for (int i = 0; i < rowNodes.length; ++i) { rowNodes[i] = new RowNodeReader(); } this.rowBuffer = new byte[rowBufferLength]; this.familyBuffer = new byte[PrefixTreeBlockMeta.MAX_FAMILY_LENGTH]; this.familyReader = new ColumnReader(familyBuffer, true); this.qualifierBuffer = new byte[qualifierBufferLength]; this.qualifierReader = new ColumnReader(qualifierBuffer, false); this.timestampDecoder = new TimestampDecoder(); this.mvccVersionDecoder = new MvccVersionDecoder(); }
@Before public void compile() throws IOException { os = new ByteArrayOutputStream(1 << 20); encoder = new PrefixTreeEncoder(os, includeMemstoreTS); inputKvs = rows.getInputs(); for (KeyValue kv : inputKvs) { encoder.write(kv); } encoder.flush(); totalBytes = encoder.getTotalBytes(); blockMetaWriter = encoder.getBlockMeta(); outputBytes = os.toByteArray(); // start reading, but save the assertions for @Test methods buffer = ByteBuffer.wrap(outputBytes); blockMetaReader = new PrefixTreeBlockMeta(buffer); searcher = new PrefixTreeArraySearcher(blockMetaReader, blockMetaReader.getRowTreeDepth(), blockMetaReader.getMaxRowLength(), blockMetaReader.getMaxQualifierLength()); searcher.initOnBlock(blockMetaReader, outputBytes, includeMemstoreTS); }
/******************* construct **************************/ public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block, int offset) { this.block = block; this.offset = offset; resetFanIndex(); this.tokenLength = UVIntTool.getInt(block, offset); this.tokenOffset = offset + UVIntTool.numBytes(tokenLength); this.fanOut = UVIntTool.getInt(block, tokenOffset + tokenLength); this.fanOffset = tokenOffset + tokenLength + UVIntTool.numBytes(fanOut); this.numCells = UVIntTool.getInt(block, fanOffset + fanOut); this.familyOffsetsOffset = fanOffset + fanOut + UVIntTool.numBytes(numCells); this.qualifierOffsetsOffset = familyOffsetsOffset + numCells * blockMeta.getFamilyOffsetWidth(); this.tagOffsetsOffset = this.qualifierOffsetsOffset + numCells * blockMeta.getQualifierOffsetWidth(); // TODO : This code may not be needed now..As we always consider tags to be present if(blockMeta.getTagsOffsetWidth() == 0) { // Make both of them same so that we know that there are no tags this.tagOffsetsOffset = this.qualifierOffsetsOffset; this.timestampIndexesOffset = qualifierOffsetsOffset + numCells * blockMeta.getQualifierOffsetWidth(); } else { this.timestampIndexesOffset = tagOffsetsOffset + numCells * blockMeta.getTagsOffsetWidth(); } this.mvccVersionIndexesOffset = timestampIndexesOffset + numCells * blockMeta.getTimestampIndexWidth(); this.operationTypesOffset = mvccVersionIndexesOffset + numCells * blockMeta.getMvccVersionIndexWidth(); this.valueOffsetsOffset = operationTypesOffset + numCells * blockMeta.getKeyValueTypeWidth(); this.valueLengthsOffset = valueOffsetsOffset + numCells * blockMeta.getValueOffsetWidth(); this.nextNodeOffsetsOffset = valueLengthsOffset + numCells * blockMeta.getValueLengthWidth(); }
public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block, boolean includeMvccVersion) { this.block = block; this.blockMeta = blockMeta; this.familyOffset = familyBuffer.length; this.familyReader.initOnBlock(blockMeta, block); this.qualifierOffset = qualifierBuffer.length; this.qualifierReader.initOnBlock(blockMeta, block); this.tagsOffset = tagsBuffer.length; this.tagsReader.initOnBlock(blockMeta, block); this.timestampDecoder.initOnBlock(blockMeta, block); this.mvccVersionDecoder.initOnBlock(blockMeta, block); this.includeMvccVersion = includeMvccVersion; resetToBeforeFirstEntry(); }
/*************** construct **************************/ public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode, ColumnNodeType nodeType) { this.blockMeta = blockMeta; this.builderNode = builderNode; this.nodeType = nodeType; calculateTokenLength(); }
/***************** construct ***********************/ public PrefixTreeEncoder(OutputStream outputStream, boolean includeMvccVersion) { // used during cell accumulation this.blockMeta = new PrefixTreeBlockMeta(); this.rowRange = new SimpleMutableByteRange(); this.familyRange = new SimpleMutableByteRange(); this.qualifierRange = new SimpleMutableByteRange(); this.timestamps = new long[INITIAL_PER_CELL_ARRAY_SIZES]; this.mvccVersions = new long[INITIAL_PER_CELL_ARRAY_SIZES]; this.typeBytes = new byte[INITIAL_PER_CELL_ARRAY_SIZES]; this.valueOffsets = new int[INITIAL_PER_CELL_ARRAY_SIZES]; this.values = new byte[VALUE_BUFFER_INIT_SIZE]; // used during compilation this.familyDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet() : new ByteRangeTreeSet(); this.qualifierDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet() : new ByteRangeTreeSet(); this.timestampEncoder = new LongEncoder(); this.mvccVersionEncoder = new LongEncoder(); this.cellTypeEncoder = new CellTypeEncoder(); this.rowTokenizer = new Tokenizer(); this.familyTokenizer = new Tokenizer(); this.qualifierTokenizer = new Tokenizer(); this.rowWriter = new RowSectionWriter(); this.familyWriter = new ColumnSectionWriter(); this.qualifierWriter = new ColumnSectionWriter(); initializeTagHelpers(); reset(outputStream, includeMvccVersion); }
@Override public void individualBlockMetaAssertions(PrefixTreeBlockMeta blockMeta) { //0: token:c; fan:d,f //1: token:f; fan:6,c //2: leaves Assert.assertEquals(3, blockMeta.getRowTreeDepth()); }
@Override public void individualBlockMetaAssertions(PrefixTreeBlockMeta blockMeta) { Assert.assertTrue(blockMeta.getNumMvccVersionBytes() > 0); Assert.assertEquals(12, blockMeta.getNumValueBytes()); Assert.assertFalse(blockMeta.isAllSameTimestamp()); Assert.assertNotNull(blockMeta.getMinTimestamp()); Assert.assertTrue(blockMeta.getTimestampIndexWidth() > 0); Assert.assertTrue(blockMeta.getTimestampDeltaWidth() > 0); Assert.assertFalse(blockMeta.isAllSameMvccVersion()); Assert.assertNotNull(blockMeta.getMinMvccVersion()); Assert.assertTrue(blockMeta.getMvccVersionIndexWidth() > 0); Assert.assertTrue(blockMeta.getMvccVersionDeltaWidth() > 0); }
/*************** construct ****************************/ public TestColumnBuilder(TestColumnData columns) { this.columns = columns; List<ByteRange> inputs = columns.getInputs(); this.columnSorter = new ByteRangeTreeSet(inputs); this.sortedUniqueColumns = columnSorter.compile().getSortedRanges(); List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns); Assert.assertTrue(Bytes.isSorted(copies)); this.blockMeta = new PrefixTreeBlockMeta(); this.blockMeta.setNumMetaBytes(0); this.blockMeta.setNumRowBytes(0); this.builder = new Tokenizer(); }
@Test public void testStreamSerialization() throws IOException { PrefixTreeBlockMeta original = createSample(); ByteArrayOutputStream os = new ByteArrayOutputStream(10000); original.writeVariableBytesToOutputStream(os); ByteBuffer buffer = ByteBuffer.wrap(os.toByteArray()); PrefixTreeBlockMeta roundTripped = new PrefixTreeBlockMeta(buffer); Assert.assertTrue(original.equals(roundTripped)); }
/***************** construct ***********************/ public PrefixTreeEncoder(OutputStream outputStream, boolean includeMvccVersion) { // used during cell accumulation this.blockMeta = new PrefixTreeBlockMeta(); this.rowRange = new SimpleByteRange(); this.familyRange = new SimpleByteRange(); this.qualifierRange = new SimpleByteRange(); this.timestamps = new long[INITIAL_PER_CELL_ARRAY_SIZES]; this.mvccVersions = new long[INITIAL_PER_CELL_ARRAY_SIZES]; this.typeBytes = new byte[INITIAL_PER_CELL_ARRAY_SIZES]; this.valueOffsets = new int[INITIAL_PER_CELL_ARRAY_SIZES]; this.values = new byte[VALUE_BUFFER_INIT_SIZE]; // used during compilation this.familyDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet() : new ByteRangeTreeSet(); this.qualifierDeduplicator = USE_HASH_COLUMN_SORTER ? new ByteRangeHashSet() : new ByteRangeTreeSet(); this.timestampEncoder = new LongEncoder(); this.mvccVersionEncoder = new LongEncoder(); this.cellTypeEncoder = new CellTypeEncoder(); this.rowTokenizer = new Tokenizer(); this.familyTokenizer = new Tokenizer(); this.qualifierTokenizer = new Tokenizer(); this.rowWriter = new RowSectionWriter(); this.familyWriter = new ColumnSectionWriter(); this.qualifierWriter = new ColumnSectionWriter(); initializeTagHelpers(); reset(outputStream, includeMvccVersion); }
