/** * @param dataBlockEncoder data block encoding algorithm to use */ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) { this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); dataBlockEncodingCtx = this.dataBlockEncoder .newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + fileContext.getBytesPerChecksum()); } baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; this.fileContext = fileContext; }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream(); baos.writeTo(stream); blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext()); }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext() .isIncludesMvcc()); PrefixTreeEncodingState state = new PrefixTreeEncodingState(); state.builder = builder; blkEncodingCtx.setEncodingState(state); }
/** * @param compressionAlgorithm compression algorithm to use * @param dataBlockEncoderAlgo data block encoding algorithm to use */ public Writer(Compression.Algorithm compressionAlgorithm, HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS, boolean includesTag) { compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm; this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; meta = new HFileContextBuilder() .withHBaseCheckSum(false) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTag) .withCompression(compressionAlgorithm) .build(); defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, DUMMY_HEADER, meta); dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext( DUMMY_HEADER, meta); baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; }
/** * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar * enough with the concept of the HFileBlockEncodingContext. */ @Override public void encodeKeyValues(ByteBuffer in, HFileBlockEncodingContext blkEncodingCtx) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(); DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder(); internalEncodeKeyValues(dataOut, in, encodingCtx.getHFileContext().isIncludesMvcc(), encodingCtx.getHFileContext().isIncludesTags()); //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE? if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) { encodingCtx.postEncoding(BlockType.ENCODED_DATA); } else { encodingCtx.postEncoding(BlockType.DATA); } }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); NoneEncoder encoder = new NoneEncoder(out, encodingCtx); NoneEncodingState state = new NoneEncodingState(); state.encoder = encoder; blkEncodingCtx.setEncodingState(state); }
/** * @param dataBlockEncoder data block encoding algorithm to use */ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) { if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + fileContext.getBytesPerChecksum()); } this.dataBlockEncoder = dataBlockEncoder != null? dataBlockEncoder: NoOpDataBlockEncoder.INSTANCE; this.dataBlockEncodingCtx = this.dataBlockEncoder. newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: This should be lazily instantiated since we usually do NOT need this default encoder this.defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: Set BAOS initial size. Use fileContext.getBlocksize() and add for header/checksum baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) { prevOffsetByType[i] = UNSET; } // TODO: Why fileContext saved away when we have dataBlockEncoder and/or // defaultDataBlockEncoder? this.fileContext = fileContext; }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
/** * @param compressionAlgorithm compression algorithm to use * @param dataBlockEncoderAlgo data block encoding algorithm to use */ public Writer(Compression.Algorithm compressionAlgorithm, HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) { compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm; this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(compressionAlgorithm, null, DUMMY_HEADER); dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext( compressionAlgorithm, DUMMY_HEADER); baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; this.includesMemstoreTS = includesMemstoreTS; }
/** * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar * enough with the concept of the HFileBlockEncodingContext. */ @Override public void encodeKeyValues(ByteBuffer in, boolean includesMvccVersion, HFileBlockEncodingContext blkEncodingCtx) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(); DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder(); internalEncodeKeyValues(dataOut, in, includesMvccVersion); //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE? if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) { encodingCtx.postEncoding(BlockType.ENCODED_DATA); } else { encodingCtx.postEncoding(BlockType.DATA); } }
/** * @param compressionAlgorithm compression algorithm to use * @param dataBlockEncoderAlgo data block encoding algorithm to use */ public Writer(Compression.Algorithm compressionAlgorithm, HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) { compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm; this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(compressionAlgorithm, null, DUMMY_HEADER); dataBlockEncodingCtx = this.dataBlockEncoder.newOnDiskDataBlockEncodingContext( compressionAlgorithm, DUMMY_HEADER); baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; this.includesMemstoreTS = includesMemstoreTS; }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext( byte[] dummyHeader, HFileContext fileContext) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder != null) { return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext); } return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext); }
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext meta) { super(dataBlockEncoder, meta); compressAlgo = meta.getCompression() == null ? NONE : meta.getCompression(); this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, DUMMY_HEADER, meta); dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext(DUMMY_HEADER, meta); baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext( DataBlockEncoding encoding, byte[] header, HFileContext meta) { if(DataBlockEncoding.PREFIX_TREE != encoding){ //i'm not sure why encoding is in the interface. Each encoder implementation should probably //know it's encoding type throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported"); } return new HFileBlockDefaultEncodingContext(encoding, header, meta); }
/** * Precondition: a non-encoded buffer. Postcondition: on-disk encoding. * * The encoded results can be stored in {@link HFileBlockEncodingContext}. * * @throws IOException */ @Override public void beforeWriteToDisk(ByteBuffer in, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (encoding == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType( in.array(), blockType); return; } encodeBufferToHFileBlockBuffer(in, encoding, encodeCtx); }
@Override public void beforeWriteToDisk(ByteBuffer in, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (!(encodeCtx.getClass().getName().equals( HFileBlockDefaultEncodingContext.class.getName()))) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + "."); } HFileBlockDefaultEncodingContext defaultContext = (HFileBlockDefaultEncodingContext) encodeCtx; defaultContext.compressAfterEncodingWithBlockType(in.array(), blockType); }
private HFileBlock createBlockOnDisk(HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); context.setDummyHeader(block.getDummyHeaderForVersion()); blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), context, block.getBlockType()); byte[] encodedBytes = context.getUncompressedBytesWithHeader(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext()); }
private void writeBlock(List<Cell> kvs, HFileContext fileContext, boolean useTags) throws IOException { HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (Cell kv : kvs) { blockEncoder.encode(kv, context, dos); } }
/** * Precondition: a non-encoded buffer. Postcondition: on-disk encoding. * * The encoded results can be stored in {@link HFileBlockEncodingContext}. * * @throws IOException */ @Override public void beforeWriteToDisk(ByteBuffer in, boolean includesMemstoreTS, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (encoding == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType( in.array(), blockType); return; } encodeBufferToHFileBlockBuffer(in, encoding, includesMemstoreTS, encodeCtx); }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext( Algorithm compressionAlgorithm, byte[] dummyHeader) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder != null) { return encoder.newDataBlockEncodingContext( compressionAlgorithm, encoding, dummyHeader); } return new HFileBlockDefaultEncodingContext( compressionAlgorithm, null, dummyHeader); }
@Override public void beforeWriteToDisk(ByteBuffer in, boolean includesMemstoreTS, HFileBlockEncodingContext encodeCtx, BlockType blockType) throws IOException { if (!(encodeCtx.getClass().getName().equals( HFileBlockDefaultEncodingContext.class.getName()))) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + "."); } HFileBlockDefaultEncodingContext defaultContext = (HFileBlockDefaultEncodingContext) encodeCtx; defaultContext.compressAfterEncodingWithBlockType(in.array(), blockType); }
/** * @param compressionAlgorithm compression algorithm to use * @param dataBlockEncoder data block encoding algorithm to use * @param checksumType type of checksum * @param bytesPerChecksum bytes per checksum */ public Writer(Compression.Algorithm compressionAlgorithm, HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS, ChecksumType checksumType, int bytesPerChecksum) { this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(compressionAlgorithm, null, HConstants.HFILEBLOCK_DUMMY_HEADER); dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext( compressionAlgorithm, HConstants.HFILEBLOCK_DUMMY_HEADER); if (bytesPerChecksum < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + bytesPerChecksum); } baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; this.includesMemstoreTS = includesMemstoreTS; this.checksumType = checksumType; this.bytesPerChecksum = bytesPerChecksum; }
private HFileBlock createBlockOnDisk(HFileBlock block) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( Compression.Algorithm.NONE, blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER); context.setDummyHeader(block.getDummyHeaderForVersion()); blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), includesMemstoreTS, context, block.getBlockType()); byte[] encodedBytes = context.getUncompressedBytesWithHeader(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); }
@Override public HFileBlockEncodingContext newDataBlockEncodingContext(Algorithm compressionAlgorithm, DataBlockEncoding encoding, byte[] header) { if(DataBlockEncoding.PREFIX_TREE != encoding){ //i'm not sure why encoding is in the interface. Each encoder implementation should probably //know it's encoding type throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported"); } return new HFileBlockDefaultEncodingContext(compressionAlgorithm, encoding, header); }
