/************* methods *****************************/ public void positionAt(int offsetIntoBlock) { this.offsetIntoBlock = offsetIntoBlock; tokenLength = UVIntTool.getInt(block, offsetIntoBlock); tokenOffsetIntoBlock = offsetIntoBlock + UVIntTool.numBytes(tokenLength); int parentStartPositionIndex = tokenOffsetIntoBlock + tokenLength; int offsetWidth; if(nodeType == ColumnNodeType.FAMILY) { offsetWidth = blockMeta.getFamilyOffsetWidth(); } else if(nodeType == ColumnNodeType.QUALIFIER) { offsetWidth = blockMeta.getQualifierOffsetWidth(); } else { offsetWidth = blockMeta.getTagsOffsetWidth(); } parentStartPosition = (int) UFIntTool.fromBytes(block, parentStartPositionIndex, offsetWidth); }
public void writeVariableBytesToOutputStream(OutputStream os) throws IOException{ UVIntTool.writeBytes(version, os); UVIntTool.writeBytes(numMetaBytes, os); UVIntTool.writeBytes(numKeyValueBytes, os); os.write(getIncludesMvccVersionByte()); UVIntTool.writeBytes(numRowBytes, os); UVIntTool.writeBytes(numFamilyBytes, os); UVIntTool.writeBytes(numQualifierBytes, os); UVIntTool.writeBytes(numTagsBytes, os); UVIntTool.writeBytes(numTimestampBytes, os); UVIntTool.writeBytes(numMvccVersionBytes, os); UVIntTool.writeBytes(numValueBytes, os); UVIntTool.writeBytes(nextNodeOffsetWidth, os); UVIntTool.writeBytes(familyOffsetWidth, os); UVIntTool.writeBytes(qualifierOffsetWidth, os); UVIntTool.writeBytes(tagsOffsetWidth, os); UVIntTool.writeBytes(timestampIndexWidth, os); UVIntTool.writeBytes(mvccVersionIndexWidth, os); UVIntTool.writeBytes(valueOffsetWidth, os); UVIntTool.writeBytes(valueLengthWidth, os); UVIntTool.writeBytes(rowTreeDepth, os); UVIntTool.writeBytes(maxRowLength, os); UVIntTool.writeBytes(maxQualifierLength, os); UVIntTool.writeBytes(maxTagsLength, os); UVLongTool.writeBytes(minTimestamp, os); UVIntTool.writeBytes(timestampDeltaWidth, os); UVLongTool.writeBytes(minMvccVersion, os); UVIntTool.writeBytes(mvccVersionDeltaWidth, os); os.write(getAllSameTypeByte()); os.write(allTypes); UVIntTool.writeBytes(numUniqueRows, os); UVIntTool.writeBytes(numUniqueFamilies, os); UVIntTool.writeBytes(numUniqueQualifiers, os); UVIntTool.writeBytes(numUniqueTags, os); }
/******************* construct **************************/ public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block, int offset) { this.block = block; this.offset = offset; resetFanIndex(); this.tokenLength = UVIntTool.getInt(block, offset); this.tokenOffset = offset + UVIntTool.numBytes(tokenLength); this.fanOut = UVIntTool.getInt(block, tokenOffset + tokenLength); this.fanOffset = tokenOffset + tokenLength + UVIntTool.numBytes(fanOut); this.numCells = UVIntTool.getInt(block, fanOffset + fanOut); this.familyOffsetsOffset = fanOffset + fanOut + UVIntTool.numBytes(numCells); this.qualifierOffsetsOffset = familyOffsetsOffset + numCells * blockMeta.getFamilyOffsetWidth(); this.tagOffsetsOffset = this.qualifierOffsetsOffset + numCells * blockMeta.getQualifierOffsetWidth(); // TODO : This code may not be needed now..As we always consider tags to be present if(blockMeta.getTagsOffsetWidth() == 0) { // Make both of them same so that we know that there are no tags this.tagOffsetsOffset = this.qualifierOffsetsOffset; this.timestampIndexesOffset = qualifierOffsetsOffset + numCells * blockMeta.getQualifierOffsetWidth(); } else { this.timestampIndexesOffset = tagOffsetsOffset + numCells * blockMeta.getTagsOffsetWidth(); } this.mvccVersionIndexesOffset = timestampIndexesOffset + numCells * blockMeta.getTimestampIndexWidth(); this.operationTypesOffset = mvccVersionIndexesOffset + numCells * blockMeta.getMvccVersionIndexWidth(); this.valueOffsetsOffset = operationTypesOffset + numCells * blockMeta.getKeyValueTypeWidth(); this.valueLengthsOffset = valueOffsetsOffset + numCells * blockMeta.getValueOffsetWidth(); this.nextNodeOffsetsOffset = valueLengthsOffset + numCells * blockMeta.getValueLengthWidth(); }
public int calculateWidthOverrideOffsetWidth(int offsetWidth){ width = 0; width += UVIntTool.numBytes(tokenWidth); width += tokenWidth; width += UVIntTool.numBytes(fanOut); width += fanOut; width += UVIntTool.numBytes(numCells); if(tokenizerNode.hasOccurrences()){ int fixedBytesPerCell = blockMeta.getFamilyOffsetWidth() + blockMeta.getQualifierOffsetWidth() + blockMeta.getTagsOffsetWidth() + blockMeta.getTimestampIndexWidth() + blockMeta.getMvccVersionIndexWidth() + blockMeta.getKeyValueTypeWidth() + blockMeta.getValueOffsetWidth() + blockMeta.getValueLengthWidth(); width += numCells * fixedBytesPerCell; } if( ! tokenizerNode.isLeaf()){ width += fanOut * offsetWidth; } return width; }
/** * UVInt: numFanBytes/fanOut * bytes: each fan byte */ public void writeFan(OutputStream os) throws IOException { UVIntTool.writeBytes(fanOut, os); if (fanOut <= 0) { return; } ArrayList<TokenizerNode> children = tokenizerNode.getChildren(); for (int i = 0; i < children.size(); ++i) { TokenizerNode child = children.get(i); os.write(child.getToken().get(0));// first byte of each child's token } }
/** * This method is called before blockMeta.qualifierOffsetWidth is known, so we pass in a * placeholder. * @param offsetWidthPlaceholder the placeholder * @return node width */ public int getWidthUsingPlaceholderForOffsetWidth(int offsetWidthPlaceholder) { int width = 0; width += UVIntTool.numBytes(tokenLength); width += token.length; width += offsetWidthPlaceholder; return width; }
public void writeBytes(OutputStream os) throws IOException { int parentOffsetWidth; if (this.nodeType == ColumnNodeType.FAMILY) { parentOffsetWidth = blockMeta.getFamilyOffsetWidth(); } else if (this.nodeType == ColumnNodeType.QUALIFIER) { parentOffsetWidth = blockMeta.getQualifierOffsetWidth(); } else { parentOffsetWidth = blockMeta.getTagsOffsetWidth(); } UVIntTool.writeBytes(tokenLength, os); os.write(token); UFIntTool.writeBytes(parentOffsetWidth, parentStartPosition, os); }
@Test public void testNumBytes() { Assert.assertEquals(1, UVIntTool.numBytes(0)); Assert.assertEquals(1, UVIntTool.numBytes(1)); Assert.assertEquals(1, UVIntTool.numBytes(100)); Assert.assertEquals(1, UVIntTool.numBytes(126)); Assert.assertEquals(1, UVIntTool.numBytes(127)); Assert.assertEquals(2, UVIntTool.numBytes(128)); Assert.assertEquals(2, UVIntTool.numBytes(129)); Assert.assertEquals(5, UVIntTool.numBytes(Integer.MAX_VALUE)); }
@Test public void testWriteBytes() throws IOException { Assert.assertArrayEquals(new byte[] { 0 }, bytesViaOutputStream(0)); Assert.assertArrayEquals(new byte[] { 1 }, bytesViaOutputStream(1)); Assert.assertArrayEquals(new byte[] { 63 }, bytesViaOutputStream(63)); Assert.assertArrayEquals(new byte[] { 127 }, bytesViaOutputStream(127)); Assert.assertArrayEquals(new byte[] { -128, 1 }, bytesViaOutputStream(128)); Assert.assertArrayEquals(new byte[] { -128 + 27, 1 }, bytesViaOutputStream(155)); Assert.assertArrayEquals(UVIntTool.MAX_VALUE_BYTES, bytesViaOutputStream(Integer.MAX_VALUE)); }
@Test public void testToBytes() { Assert.assertArrayEquals(new byte[] { 0 }, UVIntTool.getBytes(0)); Assert.assertArrayEquals(new byte[] { 1 }, UVIntTool.getBytes(1)); Assert.assertArrayEquals(new byte[] { 63 }, UVIntTool.getBytes(63)); Assert.assertArrayEquals(new byte[] { 127 }, UVIntTool.getBytes(127)); Assert.assertArrayEquals(new byte[] { -128, 1 }, UVIntTool.getBytes(128)); Assert.assertArrayEquals(new byte[] { -128 + 27, 1 }, UVIntTool.getBytes(155)); Assert.assertArrayEquals(UVIntTool.MAX_VALUE_BYTES, UVIntTool.getBytes(Integer.MAX_VALUE)); }
@Test public void testRoundTrips() { Random random = new Random(); for (int i = 0; i < 10000; ++i) { int value = random.nextInt(Integer.MAX_VALUE); byte[] bytes = UVIntTool.getBytes(value); int roundTripped = UVIntTool.getInt(bytes); Assert.assertEquals(value, roundTripped); } }
@Test public void testInputStreams() throws IOException { ByteArrayInputStream is; is = new ByteArrayInputStream(new byte[] { 0 }); Assert.assertEquals(0, UVIntTool.getInt(is)); is = new ByteArrayInputStream(new byte[] { 5 }); Assert.assertEquals(5, UVIntTool.getInt(is)); is = new ByteArrayInputStream(new byte[] { -128 + 27, 1 }); Assert.assertEquals(155, UVIntTool.getInt(is)); }
/******************* construct **************************/ public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block, int offset) { this.block = block; this.offset = offset; resetFanIndex(); this.tokenLength = UVIntTool.getInt(block, offset); this.tokenOffset = offset + UVIntTool.numBytes(tokenLength); this.fanOut = UVIntTool.getInt(block, tokenOffset + tokenLength); this.fanOffset = tokenOffset + tokenLength + UVIntTool.numBytes(fanOut); this.numCells = UVIntTool.getInt(block, fanOffset + fanOut); this.familyOffsetsOffset = fanOffset + fanOut + UVIntTool.numBytes(numCells); this.qualifierOffsetsOffset = familyOffsetsOffset + numCells * blockMeta.getFamilyOffsetWidth(); this.timestampIndexesOffset = qualifierOffsetsOffset + numCells * blockMeta.getQualifierOffsetWidth(); this.mvccVersionIndexesOffset = timestampIndexesOffset + numCells * blockMeta.getTimestampIndexWidth(); this.operationTypesOffset = mvccVersionIndexesOffset + numCells * blockMeta.getMvccVersionIndexWidth(); this.valueOffsetsOffset = operationTypesOffset + numCells * blockMeta.getKeyValueTypeWidth(); this.valueLengthsOffset = valueOffsetsOffset + numCells * blockMeta.getValueOffsetWidth(); this.nextNodeOffsetsOffset = valueLengthsOffset + numCells * blockMeta.getValueLengthWidth(); }
