@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { int klength = KeyValueUtil.keyLength(cell); int vlength = cell.getValueLength(); out.writeInt(klength); out.writeInt(vlength); CellUtil.writeFlatKey(cell, out); out.write(cell.getValueArray(), cell.getValueOffset(), vlength); int encodedKvSize = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; // Write the additional tag into the stream if (encodingCtx.getHFileContext().isIncludesTags()) { int tagsLength = cell.getTagsLength(); out.writeShort(tagsLength); if (tagsLength > 0) { out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength); } encodedKvSize += tagsLength + KeyValue.TAGS_LENGTH_SIZE; } if (encodingCtx.getHFileContext().isIncludesMvcc()) { WritableUtils.writeVLong(out, cell.getSequenceId()); encodedKvSize += WritableUtils.getVIntSize(cell.getSequenceId()); } return encodedKvSize; }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream(); baos.writeTo(stream); blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext()); }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext() .isIncludesMvcc()); PrefixTreeEncodingState state = new PrefixTreeEncodingState(); state.builder = builder; blkEncodingCtx.setEncodingState(state); }
/** * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar * enough with the concept of the HFileBlockEncodingContext. */ @Override public void encodeKeyValues(ByteBuffer in, HFileBlockEncodingContext blkEncodingCtx) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(); DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder(); internalEncodeKeyValues(dataOut, in, encodingCtx.getHFileContext().isIncludesMvcc(), encodingCtx.getHFileContext().isIncludesTags()); //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE? if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) { encodingCtx.postEncoding(BlockType.ENCODED_DATA); } else { encodingCtx.postEncoding(BlockType.DATA); } }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); NoneEncoder encoder = new NoneEncoder(out, encodingCtx); NoneEncodingState state = new NoneEncodingState(); state.encoder = encoder; blkEncodingCtx.setEncodingState(state); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
@Override public int encode(KeyValue kv, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { int klength = kv.getKeyLength(); int vlength = kv.getValueLength(); out.writeInt(klength); out.writeInt(vlength); out.write(kv.getBuffer(), kv.getKeyOffset(), klength); out.write(kv.getValueArray(), kv.getValueOffset(), vlength); int encodedKvSize = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; // Write the additional tag into the stream if (encodingCtx.getHFileContext().isIncludesTags()) { short tagsLength = kv.getTagsLength(); out.writeShort(tagsLength); if (tagsLength > 0) { out.write(kv.getTagsArray(), kv.getTagsOffset(), tagsLength); } encodedKvSize += tagsLength + KeyValue.TAGS_LENGTH_SIZE; } if (encodingCtx.getHFileContext().isIncludesMvcc()) { WritableUtils.writeVLong(out, kv.getMvccVersion()); encodedKvSize += WritableUtils.getVIntSize(kv.getMvccVersion()); } return encodedKvSize; }
/** * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar * enough with the concept of the HFileBlockEncodingContext. */ @Override public void encodeKeyValues(ByteBuffer in, boolean includesMvccVersion, HFileBlockEncodingContext blkEncodingCtx) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(); DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder(); internalEncodeKeyValues(dataOut, in, includesMvccVersion); //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE? if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) { encodingCtx.postEncoding(BlockType.ENCODED_DATA); } else { encodingCtx.postEncoding(BlockType.DATA); } }
private HFileBlock encodeDataBlock(HFileBlock block, DataBlockEncoding algo, boolean includesMemstoreTS, HFileBlockEncodingContext encodingCtx) { encodeBufferToHFileBlockBuffer( block.getBufferWithoutHeader(), algo, includesMemstoreTS, encodingCtx); byte[] encodedUncompressedBytes = encodingCtx.getUncompressedBytesWithHeader(); ByteBuffer bufferWrapper = ByteBuffer.wrap(encodedUncompressedBytes); int sizeWithoutHeader = bufferWrapper.limit() - encodingCtx.getHeaderSize(); HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA, block.getOnDiskSizeWithoutHeader(), sizeWithoutHeader, block.getPrevBlockOffset(), bufferWrapper, HFileBlock.FILL_HEADER, block.getOffset(), includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); return encodedBlock; }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext( byte[] dummyHeader, HFileContext fileContext) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder != null) { return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext); } return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext); }
@Override public void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { if (this.encoding != null && this.encoding != DataBlockEncoding.NONE) { this.encoding.getEncoder().startBlockEncoding(encodingCtx, out); } }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext( DataBlockEncoding encoding, byte[] header, HFileContext meta) { if(DataBlockEncoding.PREFIX_TREE != encoding){ //i'm not sure why encoding is in the interface. Each encoder implementation should probably //know it's encoding type throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported"); } return new HFileBlockDefaultEncodingContext(encoding, header, meta); }
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState(); PrefixTreeEncoder builder = state.builder; builder.write(cell); int size = KeyValueUtil.length(cell); if (encodingCtx.getHFileContext().isIncludesMvcc()) { size += WritableUtils.getVIntSize(cell.getSequenceId()); } return size; }
@Override public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out, byte[] uncompressedBytesWithHeader) throws IOException { PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState(); PrefixTreeEncoder builder = state.builder; builder.flush(); EncoderFactory.checkIn(builder); // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE? if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) { encodingCtx.postEncoding(BlockType.ENCODED_DATA); } else { encodingCtx.postEncoding(BlockType.DATA); } }
/** * Precondition: a non-encoded buffer. Postcondition: on-disk encoding. * * The encoded results can be stored in {@link HFileBlockEncodingContext}. * * @throws IOException */ @Override public void beforeWriteToDisk(ByteBuffer in, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (encoding == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType( in.array(), blockType); return; } encodeBufferToHFileBlockBuffer(in, encoding, encodeCtx); }
/** * Encode a block of key value pairs. * * @param in input data to encode * @param algo encoding algorithm * @param encodeCtx where will the output data be stored */ private void encodeBufferToHFileBlockBuffer(ByteBuffer in, DataBlockEncoding algo, HFileBlockEncodingContext encodeCtx) { DataBlockEncoder encoder = algo.getEncoder(); try { encoder.encodeKeyValues(in, encodeCtx); } catch (IOException e) { throw new RuntimeException(String.format( "Bug in data block encoder " + "'%s', it probably requested too much data, " + "exception message: %s.", algo.toString(), e.getMessage()), e); } }
@Override public void beforeWriteToDisk(ByteBuffer in, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (!(encodeCtx.getClass().getName().equals( HFileBlockDefaultEncodingContext.class.getName()))) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + "."); } HFileBlockDefaultEncodingContext defaultContext = (HFileBlockDefaultEncodingContext) encodeCtx; defaultContext.compressAfterEncodingWithBlockType(in.array(), blockType); }
private HFileBlock createBlockOnDisk(HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); context.setDummyHeader(block.getDummyHeaderForVersion()); blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), context, block.getBlockType()); byte[] encodedBytes = context.getUncompressedBytesWithHeader(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext()); }
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { NoneEncodingState state = (NoneEncodingState) encodingCtx .getEncodingState(); NoneEncoder encoder = state.encoder; return encoder.write(cell); }
private void writeBlock(List<Cell> kvs, HFileContext fileContext, boolean useTags) throws IOException { HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (Cell kv : kvs) { blockEncoder.encode(kv, context, dos); } }
@Override public int encode(KeyValue kv, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState(); PrefixTreeEncoder builder = state.builder; builder.write(kv); int size = kv.getLength(); if (encodingCtx.getHFileContext().isIncludesMvcc()) { size += WritableUtils.getVIntSize(kv.getMvccVersion()); } return size; }
/** * Precondition: a non-encoded buffer. Postcondition: on-disk encoding. * * The encoded results can be stored in {@link HFileBlockEncodingContext}. * * @throws IOException */ @Override public void beforeWriteToDisk(ByteBuffer in, boolean includesMemstoreTS, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (encoding == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType( in.array(), blockType); return; } encodeBufferToHFileBlockBuffer(in, encoding, includesMemstoreTS, encodeCtx); }
