1. 包列表
  2. 类列表
  3. org.apache.hadoop.hbase.io.util.StreamUtils

Java 类org.apache.hadoop.hbase.io.util.StreamUtils 实例源码

项目：ditb    文件：WALCellCodec.java    
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
  InputStream in = bs.newInput();
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
    int bytesRead = in.read(arr);
    if (bytesRead != arr.length) {
      throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
    }
    if (dict != null) dict.addEntry(arr, 0, arr.length);
    return arr;
  } else {
    // Status here is the higher-order byte of index of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    return entry;
  }
}
项目：ditb    文件：WALCellCodec.java    
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
项目：ditb    文件：ProcedureWALFormat.java    
public static void writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)
    throws IOException {
  long offset = stream.getPos();

  // Write EOF Entry
  ProcedureWALEntry.newBuilder()
    .setType(ProcedureWALEntry.Type.EOF)
    .build().writeDelimitedTo(stream);

  // Write Tracker
  tracker.writeTo(stream);

  stream.write(TRAILER_VERSION);
  StreamUtils.writeLong(stream, TRAILER_MAGIC);
  StreamUtils.writeLong(stream, offset);
}
项目：ditb    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the InputStream and writes to the destination array.
 * @param src Stream where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @throws IOException
 */
public void uncompressTags(InputStream src, byte[] dest, int offset, int length)
    throws IOException {
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = (byte) src.read();
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      int tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      IOUtils.readFully(src, dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, (byte) src.read());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      offset = Bytes.putAsShort(dest, offset, entry.length);
      System.arraycopy(entry, 0, dest, offset, entry.length);
      offset += entry.length;
    }
  }
}
项目：pbase    文件：WALCellCodec.java    
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
  InputStream in = bs.newInput();
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
    int bytesRead = in.read(arr);
    if (bytesRead != arr.length) {
      throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
    }
    if (dict != null) dict.addEntry(arr, 0, arr.length);
    return arr;
  } else {
    // Status here is the higher-order byte of index of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    return entry;
  }
}
项目：pbase    文件：WALCellCodec.java    
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
项目：pbase    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the InputStream and writes to the destination array.
 * @param src Stream where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @throws IOException
 */
public void uncompressTags(InputStream src, byte[] dest, int offset, int length)
    throws IOException {
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = (byte) src.read();
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      int tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      IOUtils.readFully(src, dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, (byte) src.read());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      offset = Bytes.putAsShort(dest, offset, entry.length);
      System.arraycopy(entry, 0, dest, offset, entry.length);
      offset += entry.length;
    }
  }
}
项目：HIndex    文件：WALCellCodec.java    
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
  InputStream in = bs.newInput();
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
    int bytesRead = in.read(arr);
    if (bytesRead != arr.length) {
      throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
    }
    if (dict != null) dict.addEntry(arr, 0, arr.length);
    return arr;
  } else {
    // Status here is the higher-order byte of index of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    return entry;
  }
}
项目：HIndex    文件：WALCellCodec.java    
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
项目：HIndex    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the InputStream and writes to the destination array.
 * @param src Stream where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @throws IOException
 */
public void uncompressTags(InputStream src, byte[] dest, int offset, short length)
    throws IOException {
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = (byte) src.read();
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      // We are writing short as tagLen. So can downcast this without any risk.
      short tagLen = (short) StreamUtils.readRawVarint32(src);
      offset = Bytes.putShort(dest, offset, tagLen);
      IOUtils.readFully(src, dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, (byte) src.read());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      offset = Bytes.putShort(dest, offset, (short) entry.length);
      System.arraycopy(entry, 0, dest, offset, entry.length);
      offset += entry.length;
    }
  }
}
项目：hbase    文件：WALCellCodec.java    
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
  InputStream in = bs.newInput();
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
    int bytesRead = in.read(arr);
    if (bytesRead != arr.length) {
      throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
    }
    if (dict != null) dict.addEntry(arr, 0, arr.length);
    return arr;
  } else {
    // Status here is the higher-order byte of index of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    return entry;
  }
}
项目：hbase    文件：WALCellCodec.java    
@Override
public void write(Cell cell) throws IOException {
  // We first write the KeyValue infrastructure as VInts.
  StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell));
  StreamUtils.writeRawVInt32(out, cell.getValueLength());
  // To support tags
  int tagsLength = cell.getTagsLength();
  StreamUtils.writeRawVInt32(out, tagsLength);
  PrivateCellUtil.compressRow(out, cell, compression.rowDict);
  PrivateCellUtil.compressFamily(out, cell, compression.familyDict);
  PrivateCellUtil.compressQualifier(out, cell, compression.qualifierDict);
  // Write timestamp, type and value as uncompressed.
  StreamUtils.writeLong(out, cell.getTimestamp());
  out.write(cell.getTypeByte());
  PrivateCellUtil.writeValue(out, cell, cell.getValueLength());
  if (tagsLength > 0) {
    if (compression.tagCompressionContext != null) {
      // Write tags using Dictionary compression
      PrivateCellUtil.compressTags(out, cell, compression.tagCompressionContext);
    } else {
      // Tag compression is disabled within the WAL compression. Just write the tags bytes as
      // it is.
      PrivateCellUtil.writeTags(out, cell, tagsLength);
    }
  }
}
项目：hbase    文件：WALCellCodec.java    
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
项目：hbase    文件：ProcedureWALFormat.java    
public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)
    throws IOException {
  long offset = stream.getPos();

  // Write EOF Entry
  ProcedureWALEntry.newBuilder()
    .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)
    .build().writeDelimitedTo(stream);

  // Write Tracker
  tracker.toProto().writeDelimitedTo(stream);

  stream.write(TRAILER_VERSION);
  StreamUtils.writeLong(stream, TRAILER_MAGIC);
  StreamUtils.writeLong(stream, offset);
  return stream.getPos() - offset;
}
项目：hbase    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the InputStream and writes to the destination array.
 * @param src Stream where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @throws IOException
 */
public void uncompressTags(InputStream src, byte[] dest, int offset, int length)
    throws IOException {
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = (byte) src.read();
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      int tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      IOUtils.readFully(src, dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, (byte) src.read());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      offset = Bytes.putAsShort(dest, offset, entry.length);
      System.arraycopy(entry, 0, dest, offset, entry.length);
      offset += entry.length;
    }
  }
}
项目：PyroDB    文件：WALCellCodec.java    
private static byte[] uncompressByteString(ByteString bs, Dictionary dict) throws IOException {
  InputStream in = bs.newInput();
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    byte[] arr = new byte[StreamUtils.readRawVarint32(in)];
    int bytesRead = in.read(arr);
    if (bytesRead != arr.length) {
      throw new IOException("Cannot read; wanted " + arr.length + ", but got " + bytesRead);
    }
    if (dict != null) dict.addEntry(arr, 0, arr.length);
    return arr;
  } else {
    // Status here is the higher-order byte of index of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    return entry;
  }
}
项目：PyroDB    文件：WALCellCodec.java    
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
项目：PyroDB    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the InputStream and writes to the destination array.
 * @param src Stream where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @throws IOException
 */
public void uncompressTags(InputStream src, byte[] dest, int offset, short length)
    throws IOException {
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = (byte) src.read();
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      // We are writing short as tagLen. So can downcast this without any risk.
      short tagLen = (short) StreamUtils.readRawVarint32(src);
      offset = Bytes.putShort(dest, offset, tagLen);
      IOUtils.readFully(src, dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, (byte) src.read());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      offset = Bytes.putShort(dest, offset, (short) entry.length);
      System.arraycopy(entry, 0, dest, offset, entry.length);
      offset += entry.length;
    }
  }
}
项目：ditb    文件：DefaultVisibilityLabelServiceImpl.java    
private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag)
    throws IOException {
  List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>();
  int offset = tag.getTagOffset();
  int endOffset = offset + tag.getTagLength();
  while (offset < endOffset) {
    Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
    tagsOrdinalInSortedOrder.add(result.getFirst());
    offset += result.getSecond();
  }
  Collections.sort(tagsOrdinalInSortedOrder);
  fullTagsList.add(tagsOrdinalInSortedOrder);
}
项目：ditb    文件：WALCellCodec.java    
@Override
public void write(Cell cell) throws IOException {
  // We first write the KeyValue infrastructure as VInts.
  StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell));
  StreamUtils.writeRawVInt32(out, cell.getValueLength());
  // To support tags
  int tagsLength = cell.getTagsLength();
  StreamUtils.writeRawVInt32(out, tagsLength);

  // Write row, qualifier, and family; use dictionary
  // compression as they're likely to have duplicates.
  write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), compression.rowDict);
  write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
      compression.familyDict);
  write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
      compression.qualifierDict);

  // Write timestamp, type and value as uncompressed.
  StreamUtils.writeLong(out, cell.getTimestamp());
  out.write(cell.getTypeByte());
  out.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
  if (tagsLength > 0) {
    if (compression.tagCompressionContext != null) {
      // Write tags using Dictionary compression
      compression.tagCompressionContext.compressTags(out, cell.getTagsArray(),
          cell.getTagsOffset(), tagsLength);
    } else {
      // Tag compression is disabled within the WAL compression. Just write the tags bytes as
      // it is.
      out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
    }
  }
}
项目：ditb    文件：WALCellCodec.java    
private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (dict != null) {
    dictIdx = dict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：ditb    文件：ProcedureWALFormat.java    
public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)
    throws IOException {
  long trailerPos = size - 17; // Beginning of the Trailer Jump

  if (trailerPos < startPos) {
    throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);
  }

  stream.seek(trailerPos);
  int version = stream.read();
  if (version != TRAILER_VERSION) {
    throw new InvalidWALDataException("Invalid Trailer version. got " + version +
        " expected " + TRAILER_VERSION);
  }

  long magic = StreamUtils.readLong(stream);
  if (magic != TRAILER_MAGIC) {
    throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +
        " expected " + TRAILER_MAGIC);
  }

  long trailerOffset = StreamUtils.readLong(stream);
  stream.seek(trailerOffset);

  ProcedureWALEntry entry = readEntry(stream);
  if (entry.getType() != ProcedureWALEntry.Type.EOF) {
    throw new InvalidWALDataException("Invalid Trailer begin");
  }

  ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()
    .setVersion(version)
    .setTrackerPos(stream.getPos())
    .build();
  return trailer;
}
项目：ditb    文件：KeyValue.java    
/**
 * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
 * not require a {@link DataOutput}, just take plain {@link OutputStream}
 * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
 * @param kv
 * @param out
 * @param withTags
 * @return Length written on stream
 * @throws IOException
 * @see #create(DataInput) for the inverse function
 * @see #write(KeyValue, DataOutput)
 * @see KeyValueUtil#oswrite(Cell, OutputStream, boolean)
 */
public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags)
    throws IOException {
  // In KeyValueUtil#oswrite we do a Cell serialization as KeyValue. Any changes doing here, pls
  // check KeyValueUtil#oswrite also and do necessary changes.
  int length = kv.getLength();
  if (!withTags) {
    length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
  }
  // This does same as DataOuput#writeInt (big-endian, etc.)
  StreamUtils.writeInt(out, length);
  out.write(kv.getBuffer(), kv.getOffset(), length);
  return length + Bytes.SIZEOF_INT;
}
项目：ditb    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the input ByteBuffer and writes to the destination array.
 * @param src Buffer where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @return bytes count read from source to uncompress all tags.
 * @throws IOException
 */
public int uncompressTags(ByteBuffer src, byte[] dest, int offset, int length)
    throws IOException {
  int srcBeginPos = src.position();
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = src.get();
    int tagLen;
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      src.get(dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, src.get());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      tagLen = entry.length;
      offset = Bytes.putAsShort(dest, offset, tagLen);
      System.arraycopy(entry, 0, dest, offset, tagLen);
      offset += tagLen;
    }
  }
  return src.position() - srcBeginPos;
}
项目：ditb    文件：TagCompressionContext.java    
private void write(byte[] data, int offset, int length, OutputStream out) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (tagDict != null) {
    dictIdx = tagDict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：pbase    文件：DefaultVisibilityLabelServiceImpl.java    
private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag)
    throws IOException {
  List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>();
  int offset = tag.getTagOffset();
  int endOffset = offset + tag.getTagLength();
  while (offset < endOffset) {
    Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
    tagsOrdinalInSortedOrder.add(result.getFirst());
    offset += result.getSecond();
  }
  Collections.sort(tagsOrdinalInSortedOrder);
  fullTagsList.add(tagsOrdinalInSortedOrder);
}
项目：pbase    文件：WALCellCodec.java    
@Override
public void write(Cell cell) throws IOException {
  // We first write the KeyValue infrastructure as VInts.
  StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell));
  StreamUtils.writeRawVInt32(out, cell.getValueLength());
  // To support tags
  int tagsLength = cell.getTagsLength();
  StreamUtils.writeRawVInt32(out, tagsLength);

  // Write row, qualifier, and family; use dictionary
  // compression as they're likely to have duplicates.
  write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), compression.rowDict);
  write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
      compression.familyDict);
  write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
      compression.qualifierDict);

  // Write timestamp, type and value as uncompressed.
  StreamUtils.writeLong(out, cell.getTimestamp());
  out.write(cell.getTypeByte());
  out.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
  if (tagsLength > 0) {
    if (compression.tagCompressionContext != null) {
      // Write tags using Dictionary compression
      compression.tagCompressionContext.compressTags(out, cell.getTagsArray(),
          cell.getTagsOffset(), tagsLength);
    } else {
      // Tag compression is disabled within the WAL compression. Just write the tags bytes as
      // it is.
      out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
    }
  }
}
项目：pbase    文件：WALCellCodec.java    
private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (dict != null) {
    dictIdx = dict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：pbase    文件：KeyValue.java    
/**
 * Write out a KeyValue in the manner in which we used to when KeyValue was a Writable but do
 * not require a {@link DataOutput}, just take plain {@link OutputStream}
 * Named <code>oswrite</code> so does not clash with {@link #write(KeyValue, DataOutput)}
 * @param kv
 * @param out
 * @param withTags
 * @return Length written on stream
 * @throws IOException
 * @see #create(DataInput) for the inverse function
 * @see #write(KeyValue, DataOutput)
 * @see KeyValueUtil#oswrite(Cell, OutputStream, boolean)
 */
public static long oswrite(final KeyValue kv, final OutputStream out, final boolean withTags)
    throws IOException {
  // In KeyValueUtil#oswrite we do a Cell serialization as KeyValue. Any changes doing here, pls
  // check KeyValueUtil#oswrite also and do necessary changes.
  int length = kv.getLength();
  if (!withTags) {
    length = kv.getKeyLength() + kv.getValueLength() + KEYVALUE_INFRASTRUCTURE_SIZE;
  }
  // This does same as DataOuput#writeInt (big-endian, etc.)
  StreamUtils.writeInt(out, length);
  out.write(kv.getBuffer(), kv.getOffset(), length);
  return length + Bytes.SIZEOF_INT;
}
项目：pbase    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the input ByteBuffer and writes to the destination array.
 * @param src Buffer where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @return bytes count read from source to uncompress all tags.
 * @throws IOException
 */
public int uncompressTags(ByteBuffer src, byte[] dest, int offset, int length)
    throws IOException {
  int srcBeginPos = src.position();
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = src.get();
    int tagLen;
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      src.get(dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, src.get());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      tagLen = entry.length;
      offset = Bytes.putAsShort(dest, offset, tagLen);
      System.arraycopy(entry, 0, dest, offset, tagLen);
      offset += tagLen;
    }
  }
  return src.position() - srcBeginPos;
}
项目：pbase    文件：TagCompressionContext.java    
private void write(byte[] data, int offset, int length, OutputStream out) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (tagDict != null) {
    dictIdx = tagDict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：HIndex    文件：VisibilityController.java    
private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos)
    throws IOException, InvalidLabelException {
  if (node.isSingleNode()) {
    String identifier = null;
    int labelOrdinal = 0;
    if (node instanceof LeafExpressionNode) {
      identifier = ((LeafExpressionNode) node)
          .getIdentifier();
      if (LOG.isTraceEnabled()) {
        LOG.trace("The identifier is "+identifier);
      }
      labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier);
    } else {
      // This is a NOT node.
      LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
          .getChildExps().get(0);
      identifier = lNode.getIdentifier();
      labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier);
      labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
    }
    if (labelOrdinal == 0) {
      throw new InvalidLabelException("Invalid visibility label " + identifier);
    }
    StreamUtils.writeRawVInt32(dos, labelOrdinal);
  } else {
    List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
    for (ExpressionNode child : childExps) {
      writeLabelOrdinalsToStream(child, dos);
    }
  }
}
项目：HIndex    文件：WALCellCodec.java    
private void write(byte[] data, int offset, int length, Dictionary dict) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (dict != null) {
    dictIdx = dict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：HIndex    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the input ByteBuffer and writes to the destination array.
 * @param src Buffer where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @return bytes count read from source to uncompress all tags.
 * @throws IOException
 */
public int uncompressTags(ByteBuffer src, byte[] dest, int offset, int length)
    throws IOException {
  int srcBeginPos = src.position();
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = src.get();
    short tagLen;
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      // We are writing short as tagLen. So can downcast this without any risk.
      tagLen = (short) StreamUtils.readRawVarint32(src);
      offset = Bytes.putShort(dest, offset, tagLen);
      src.get(dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, src.get());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      tagLen = (short) entry.length;
      offset = Bytes.putShort(dest, offset, tagLen);
      System.arraycopy(entry, 0, dest, offset, tagLen);
      offset += tagLen;
    }
  }
  return src.position() - srcBeginPos;
}
项目：HIndex    文件：TagCompressionContext.java    
private void write(byte[] data, int offset, short length, OutputStream out) throws IOException {
  short dictIdx = Dictionary.NOT_IN_DICTIONARY;
  if (tagDict != null) {
    dictIdx = tagDict.findEntry(data, offset, length);
  }
  if (dictIdx == Dictionary.NOT_IN_DICTIONARY) {
    out.write(Dictionary.NOT_IN_DICTIONARY);
    StreamUtils.writeRawVInt32(out, length);
    out.write(data, offset, length);
  } else {
    StreamUtils.writeShort(out, dictIdx);
  }
}
项目：hbase    文件：ProcedureWALFormat.java    
public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)
    throws IOException {
  // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset
  long trailerPos = size - 17;

  if (trailerPos < startPos) {
    throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);
  }

  stream.seek(trailerPos);
  int version = stream.read();
  if (version != TRAILER_VERSION) {
    throw new InvalidWALDataException("Invalid Trailer version. got " + version +
        " expected " + TRAILER_VERSION);
  }

  long magic = StreamUtils.readLong(stream);
  if (magic != TRAILER_MAGIC) {
    throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +
        " expected " + TRAILER_MAGIC);
  }

  long trailerOffset = StreamUtils.readLong(stream);
  stream.seek(trailerOffset);

  ProcedureWALEntry entry = readEntry(stream);
  if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {
    throw new InvalidWALDataException("Invalid Trailer begin");
  }

  ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()
    .setVersion(version)
    .setTrackerPos(stream.getPos())
    .build();
  return trailer;
}
项目：hbase    文件：PrivateCellUtil.java    
public static int writeFlatKey(Cell cell, OutputStream out) throws IOException {
  short rowLen = cell.getRowLength();
  byte fLen = cell.getFamilyLength();
  int qLen = cell.getQualifierLength();
  // Using just one if/else loop instead of every time checking before writing every
  // component of cell
  if (cell instanceof ByteBufferExtendedCell) {
    StreamUtils.writeShort(out, rowLen);
    ByteBufferUtils.copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getRowByteBuffer(),
      ((ByteBufferExtendedCell) cell).getRowPosition(), rowLen);
    out.write(fLen);
    ByteBufferUtils.copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getFamilyByteBuffer(),
      ((ByteBufferExtendedCell) cell).getFamilyPosition(), fLen);
    ByteBufferUtils
      .copyBufferToStream(out, ((ByteBufferExtendedCell) cell).getQualifierByteBuffer(),
        ((ByteBufferExtendedCell) cell).getQualifierPosition(), qLen);
  } else {
    StreamUtils.writeShort(out, rowLen);
    out.write(cell.getRowArray(), cell.getRowOffset(), rowLen);
    out.write(fLen);
    out.write(cell.getFamilyArray(), cell.getFamilyOffset(), fLen);
    out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qLen);
  }
  StreamUtils.writeLong(out, cell.getTimestamp());
  out.write(cell.getTypeByte());
  return Bytes.SIZEOF_SHORT + rowLen + Bytes.SIZEOF_BYTE + fLen + qLen + Bytes.SIZEOF_LONG
    + Bytes.SIZEOF_BYTE;
}
项目：hbase    文件：ByteBufferUtils.java    
/**
 * Put in output stream 32 bit integer (Big Endian byte order).
 * @param out Where to put integer.
 * @param value Value of integer.
 * @throws IOException On stream error.
 */
public static void putInt(OutputStream out, final int value)
    throws IOException {
  // We have writeInt in ByteBufferOutputStream so that it can directly write
  // int to underlying
  // ByteBuffer in one step.
  if (out instanceof ByteBufferWriter) {
    ((ByteBufferWriter) out).writeInt(value);
  } else {
    StreamUtils.writeInt(out, value);
  }
}
项目：hbase    文件：TagCompressionContext.java    
/**
 * Uncompress tags from the input ByteBuffer and writes to the destination array.
 * @param src Buffer where the compressed tags are available
 * @param dest Destination array where to write the uncompressed tags
 * @param offset Offset in destination where tags to be written
 * @param length Length of all tag bytes
 * @return bytes count read from source to uncompress all tags.
 * @throws IOException
 */
public int uncompressTags(ByteBuff src, byte[] dest, int offset, int length)
    throws IOException {
  int srcBeginPos = src.position();
  int endOffset = offset + length;
  while (offset < endOffset) {
    byte status = src.get();
    int tagLen;
    if (status == Dictionary.NOT_IN_DICTIONARY) {
      tagLen = StreamUtils.readRawVarint32(src);
      offset = Bytes.putAsShort(dest, offset, tagLen);
      src.get(dest, offset, tagLen);
      tagDict.addEntry(dest, offset, tagLen);
      offset += tagLen;
    } else {
      short dictIdx = StreamUtils.toShort(status, src.get());
      byte[] entry = tagDict.getEntry(dictIdx);
      if (entry == null) {
        throw new IOException("Missing dictionary entry for index " + dictIdx);
      }
      tagLen = entry.length;
      offset = Bytes.putAsShort(dest, offset, tagLen);
      System.arraycopy(entry, 0, dest, offset, tagLen);
      offset += tagLen;
    }
  }
  return src.position() - srcBeginPos;
}
项目：hbase    文件：BufferedDataBlockEncoder.java    
@Override
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
    throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException (this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " +
        "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx =
      (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding(out);
  if (encodingCtx.getHFileContext().isIncludesTags()
      && encodingCtx.getHFileContext().isCompressTags()) {
    if (encodingCtx.getTagCompressionContext() != null) {
      // It will be overhead to create the TagCompressionContext again and again for every block
      // encoding.
      encodingCtx.getTagCompressionContext().clear();
    } else {
      try {
        TagCompressionContext tagCompressionContext = new TagCompressionContext(
            LRUDictionary.class, Byte.MAX_VALUE);
        encodingCtx.setTagCompressionContext(tagCompressionContext);
      } catch (Exception e) {
        throw new IOException("Failed to initialize TagCompressionContext", e);
      }
    }
  }
  StreamUtils.writeInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding()
  blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState());
}