Java 类org.apache.lucene.analysis.cjk.CJKWidthFilter 实例源码

public void testKatakanaReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
  );
}

public void testRomajiReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
  );
}

public void testKatakanaReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
  );
}

public void testRomajiReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
  );
}

public void testKatakanaReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "コンヤ", "ハ", "ロバート", "センセイ", "ト", "ハナシ", "タ" }
  );
}

public void testRomajiReadingsHalfWidth() throws IOException {
  Analyzer a = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
      TokenStream stream = new CJKWidthFilter(tokenizer);
      return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
    }
  };
  assertAnalyzesTo(a, "今夜はﾛﾊﾞｰﾄ先生と話した",
      new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
  );
}

@Override
public TokenStream create(TokenStream tokenStream) {
    return new CJKWidthFilter(tokenStream);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream tokenStream) {
    return new CJKWidthFilter(tokenStream);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}

@Override
public TokenStream create(TokenStream input) {
  return new CJKWidthFilter(input);
}