Java 类org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode 实例源码

public void reset(TokenizerNode node) {
  this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
  this.tokenizerNode = node;
  this.tokenWidth = 0;
  this.fanOut = 0;
  this.numCells = 0;
  this.width = 0;
  calculateOffsetsAndLengths();
}

/**
 * UVInt: numFanBytes/fanOut
 * bytes: each fan byte
 */
public void writeFan(OutputStream os) throws IOException {
  UVIntTool.writeBytes(fanOut, os);
  if (fanOut <= 0) {
    return;
  }
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    os.write(child.getToken().get(0));// first byte of each child's token
  }
}

/**
 * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
 */
protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
    UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
  }
}

protected RowNodeWriter initializeWriter(List<RowNodeWriter> list, int index,
    TokenizerNode builderNode) {
  RowNodeWriter rowNodeWriter = null;
  //check if there is an existing node we can recycle
  if (index >= list.size()) {
    //there are not enough existing nodes, so add a new one which will be retrieved below
    list.add(new RowNodeWriter(prefixTreeEncoder, builderNode));
  }
  rowNodeWriter = list.get(index);
  rowNodeWriter.reset(builderNode);
  return rowNodeWriter;
}

/***************** static ******************************/

  protected static ArrayList<TokenizerNode> filterByLeafAndReverse(
      ArrayList<TokenizerNode> ins, boolean leaves) {
    ArrayList<TokenizerNode> outs = Lists.newArrayList();
    for (int i = ins.size() - 1; i >= 0; --i) {
      TokenizerNode n = ins.get(i);
      if (n.isLeaf() && leaves || (!n.isLeaf() && !leaves)) {
        outs.add(ins.get(i));
      }
    }
    return outs;
  }

/*************** construct **************************/

  public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode,
      ColumnNodeType nodeType) {
    this.blockMeta = blockMeta;
    this.builderNode = builderNode;
    this.nodeType = nodeType;
    calculateTokenLength();
  }

@Test
public void testSearching() {
  for (byte[] input : inputs) {
    TokenizerRowSearchResult resultHolder = new TokenizerRowSearchResult();
    builder.getNode(resultHolder, input, 0, input.length);
    TokenizerNode n = resultHolder.getMatchingNode();
    byte[] output = n.getNewByteArray();
    Assert.assertTrue(Bytes.equals(input, output));
  }
}

public void reset(TokenizerNode node) {
  this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
  this.tokenizerNode = node;
  this.tokenWidth = 0;
  this.fanOut = 0;
  this.numCells = 0;
  this.width = 0;
  calculateOffsetsAndLengths();
}

/**
 * UVInt: numFanBytes/fanOut
 * bytes: each fan byte
 */
public void writeFan(OutputStream os) throws IOException {
  UVIntTool.writeBytes(fanOut, os);
  if (fanOut <= 0) {
    return;
  }
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    os.write(child.getToken().get(0));// first byte of each child's token
  }
}

/**
 * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
 */
protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
    UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
  }
}

protected RowNodeWriter initializeWriter(List<RowNodeWriter> list, int index,
    TokenizerNode builderNode) {
  RowNodeWriter rowNodeWriter = null;
  //check if there is an existing node we can recycle
  if (index >= list.size()) {
    //there are not enough existing nodes, so add a new one which will be retrieved below
    list.add(new RowNodeWriter(prefixTreeEncoder, builderNode));
  }
  rowNodeWriter = list.get(index);
  rowNodeWriter.reset(builderNode);
  return rowNodeWriter;
}

/***************** static ******************************/

  protected static ArrayList<TokenizerNode> filterByLeafAndReverse(
      ArrayList<TokenizerNode> ins, boolean leaves) {
    ArrayList<TokenizerNode> outs = Lists.newArrayList();
    for (int i = ins.size() - 1; i >= 0; --i) {
      TokenizerNode n = ins.get(i);
      if (n.isLeaf() && leaves || (!n.isLeaf() && !leaves)) {
        outs.add(ins.get(i));
      }
    }
    return outs;
  }

/*************** construct **************************/

  public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode,
      ColumnNodeType nodeType) {
    this.blockMeta = blockMeta;
    this.builderNode = builderNode;
    this.nodeType = nodeType;
    calculateTokenLength();
  }

@Test
public void testSearching() {
  for (byte[] input : inputs) {
    TokenizerRowSearchResult resultHolder = new TokenizerRowSearchResult();
    builder.getNode(resultHolder, input, 0, input.length);
    TokenizerNode n = resultHolder.getMatchingNode();
    byte[] output = n.getNewByteArray();
    Assert.assertTrue(Bytes.equals(input, output));
  }
}

public void reset(TokenizerNode node) {
  this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
  this.tokenizerNode = node;
  this.tokenWidth = 0;
  this.fanOut = 0;
  this.numCells = 0;
  this.width = 0;
  calculateOffsetsAndLengths();
}

/**
 * UVInt: numFanBytes/fanOut
 * bytes: each fan byte
 */
public void writeFan(OutputStream os) throws IOException {
  UVIntTool.writeBytes(fanOut, os);
  if (fanOut <= 0) {
    return;
  }
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    os.write(child.getToken().get(0));// first byte of each child's token
  }
}

/**
 * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
 */
protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
    UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
  }
}

protected RowNodeWriter initializeWriter(List<RowNodeWriter> list, int index,
    TokenizerNode builderNode) {
  RowNodeWriter rowNodeWriter = null;
  //check if there is an existing node we can recycle
  if (index >= list.size()) {
    //there are not enough existing nodes, so add a new one which will be retrieved below
    list.add(new RowNodeWriter(prefixTreeEncoder, builderNode));
  }
  rowNodeWriter = list.get(index);
  rowNodeWriter.reset(builderNode);
  return rowNodeWriter;
}

/***************** static ******************************/

  protected static ArrayList<TokenizerNode> filterByLeafAndReverse(
      ArrayList<TokenizerNode> ins, boolean leaves) {
    ArrayList<TokenizerNode> outs = Lists.newArrayList();
    for (int i = ins.size() - 1; i >= 0; --i) {
      TokenizerNode n = ins.get(i);
      if (n.isLeaf() && leaves || (!n.isLeaf() && !leaves)) {
        outs.add(ins.get(i));
      }
    }
    return outs;
  }

/*************** construct **************************/

  public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode,
      ColumnNodeType nodeType) {
    this.blockMeta = blockMeta;
    this.builderNode = builderNode;
    this.nodeType = nodeType;
    calculateTokenLength();
  }

@Test
public void testSearching() {
  for (byte[] input : inputs) {
    TokenizerRowSearchResult resultHolder = new TokenizerRowSearchResult();
    builder.getNode(resultHolder, input, 0, input.length);
    TokenizerNode n = resultHolder.getMatchingNode();
    byte[] output = n.getNewByteArray();
    Assert.assertTrue(Bytes.equals(input, output));
  }
}

public void reset(TokenizerNode node) {
  this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
  this.tokenizerNode = node;
  this.tokenWidth = 0;
  this.fanOut = 0;
  this.numCells = 0;
  this.width = 0;
  calculateOffsetsAndLengths();
}

/**
 * UVInt: numFanBytes/fanOut
 * bytes: each fan byte
 */
public void writeFan(OutputStream os) throws IOException {
  UVIntTool.writeBytes(fanOut, os);
  if (fanOut <= 0) {
    return;
  }
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    os.write(child.getToken().get(0));// first byte of each child's token
  }
}

/**
 * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
 */
protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
    UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
  }
}

protected RowNodeWriter initializeWriter(List<RowNodeWriter> list, int index,
    TokenizerNode builderNode) {
  RowNodeWriter rowNodeWriter = null;
  //check if there is an existing node we can recycle
  if (index >= list.size()) {
    //there are not enough existing nodes, so add a new one which will be retrieved below
    list.add(new RowNodeWriter(prefixTreeEncoder, builderNode));
  }
  rowNodeWriter = list.get(index);
  rowNodeWriter.reset(builderNode);
  return rowNodeWriter;
}

/***************** static ******************************/

  protected static ArrayList<TokenizerNode> filterByLeafAndReverse(
      ArrayList<TokenizerNode> ins, boolean leaves) {
    ArrayList<TokenizerNode> outs = Lists.newArrayList();
    for (int i = ins.size() - 1; i >= 0; --i) {
      TokenizerNode n = ins.get(i);
      if (n.isLeaf() && leaves || (!n.isLeaf() && !leaves)) {
        outs.add(ins.get(i));
      }
    }
    return outs;
  }

/*************** construct **************************/

  public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode,
      ColumnNodeType nodeType) {
    this.blockMeta = blockMeta;
    this.builderNode = builderNode;
    this.nodeType = nodeType;
    calculateTokenLength();
  }

@Test
public void testSearching() {
  for (byte[] input : inputs) {
    TokenizerRowSearchResult resultHolder = new TokenizerRowSearchResult();
    builder.getNode(resultHolder, input, 0, input.length);
    TokenizerNode n = resultHolder.getMatchingNode();
    byte[] output = n.getNewByteArray();
    Assert.assertTrue(Bytes.equals(input, output));
  }
}

public void reset(TokenizerNode node) {
  this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
  this.tokenizerNode = node;
  this.tokenWidth = 0;
  this.fanOut = 0;
  this.numCells = 0;
  this.width = 0;
  calculateOffsetsAndLengths();
}

/**
 * UVInt: numFanBytes/fanOut
 * bytes: each fan byte
 */
public void writeFan(OutputStream os) throws IOException {
  UVIntTool.writeBytes(fanOut, os);
  if (fanOut <= 0) {
    return;
  }
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    os.write(child.getToken().get(0));// first byte of each child's token
  }
}

/**
 * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
 */
protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
  ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
  for (int i = 0; i < children.size(); ++i) {
    TokenizerNode child = children.get(i);
    int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
    UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
  }
}

protected RowNodeWriter initializeWriter(List<RowNodeWriter> list, int index,
    TokenizerNode builderNode) {
  RowNodeWriter rowNodeWriter = null;
  //check if there is an existing node we can recycle
  if (index >= list.size()) {
    //there are not enough existing nodes, so add a new one which will be retrieved below
    list.add(new RowNodeWriter(prefixTreeEncoder, builderNode));
  }
  rowNodeWriter = list.get(index);
  rowNodeWriter.reset(builderNode);
  return rowNodeWriter;
}

/***************** static ******************************/

  protected static ArrayList<TokenizerNode> filterByLeafAndReverse(
      ArrayList<TokenizerNode> ins, boolean leaves) {
    ArrayList<TokenizerNode> outs = Lists.newArrayList();
    for (int i = ins.size() - 1; i >= 0; --i) {
      TokenizerNode n = ins.get(i);
      if (n.isLeaf() && leaves || (!n.isLeaf() && !leaves)) {
        outs.add(ins.get(i));
      }
    }
    return outs;
  }

/*************** construct **************************/

  public ColumnNodeWriter(PrefixTreeBlockMeta blockMeta, TokenizerNode builderNode,
      boolean familyVsQualifier) {
    this.blockMeta = blockMeta;
    this.builderNode = builderNode;
    this.familyVsQualifier = familyVsQualifier;
    calculateTokenLength();
  }

@Test
public void testSearching() {
  for (byte[] input : inputs) {
    TokenizerRowSearchResult resultHolder = new TokenizerRowSearchResult();
    builder.getNode(resultHolder, input, 0, input.length);
    TokenizerNode n = resultHolder.getMatchingNode();
    byte[] output = n.getNewByteArray();
    Assert.assertTrue(Bytes.equals(input, output));
  }
}

/*********************** construct *************************/

  public RowNodeWriter(PrefixTreeEncoder keyValueBuilder, TokenizerNode tokenizerNode) {
    reconstruct(keyValueBuilder, tokenizerNode);
  }

public void reconstruct(PrefixTreeEncoder prefixTreeEncoder, TokenizerNode tokenizerNode) {
  this.prefixTreeEncoder = prefixTreeEncoder;
  reset(tokenizerNode);
}

public ArrayList<TokenizerNode> getNonLeaves() {
  return nonLeaves;
}

public ArrayList<TokenizerNode> getLeaves() {
  return leaves;
}

public ArrayList<TokenizerNode> getNonLeaves() {
  return nonLeaves;
}