@Override public void copyTo(AttributeImpl target) { if (target instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target; to.copyBuffer(buffer(), 0, length()); to.positionIncrement = positionIncrement; to.positionLength = positionLength; to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); } }
@Override public void copyTo(AttributeImpl target) { if (target instanceof Token) { final Token to = (Token) target; to.reinit(this); // reinit shares the payload, so clone it: if (payload !=null) { to.payload = payload.clone(); } } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PayloadAttribute) target).setPayload((payload == null) ? null : payload.clone()); ((FlagsAttribute) target).setFlags(flags); ((TypeAttribute) target).setType(type); } }
/** * Returns the list of tokens extracted from the query string using the specified analyzer. * * @param field document field. * * @param queryTerms query string. * * @param distinctTokens if true, return the distinct tokens in the query string. * * @return the list of tokens extracted from the given query. * * @throws IOException */ List<String> getTokens(String field, String queryTerms, boolean distinctTokens) throws IOException { List<String> tokens = new ArrayList<String>(); StringReader topicTitleReader = new StringReader(queryTerms); Set<String> seenTokens = new TreeSet<String>(); TokenStream tok; tok = analyzer.tokenStream(field, topicTitleReader); tok.reset(); while (tok.incrementToken()) { Iterator<AttributeImpl> atts = tok.getAttributeImplsIterator(); AttributeImpl token = atts.next(); String text = "" + token; if (seenTokens.contains(text) && distinctTokens) { continue; } seenTokens.add(text); tokens.add(text); } tok.close(); return tokens; }
/** * We just want to create a new array if the length differs. */ @Override public void copyTo(AttributeImpl input) { StemmingBufferAttributeImpl copyAttributeImpl = (StemmingBufferAttributeImpl) input; if (copyAttributeImpl.stemmedToken.length < stemmedTokenLength) { copyAttributeImpl.stemmedToken = new char[stemmedTokenLength]; } System.arraycopy(stemmedToken, 0, copyAttributeImpl.stemmedToken, 0, stemmedTokenLength); if (copyAttributeImpl.originalToken.length < originalTokenLength) { copyAttributeImpl.originalToken = new char[originalTokenLength]; } System.arraycopy(originalToken, 0, copyAttributeImpl.originalToken, 0, originalTokenLength); copyAttributeImpl.stemmedTokenLength = stemmedTokenLength; copyAttributeImpl.originalTokenLength = originalTokenLength; copyAttributeImpl.stemmedTokenHasBeenEmitted = stemmedTokenHasBeenEmitted; }
@Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, reader) { final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class); int p = 0; @Override public boolean incrementToken() { if( p >= tokens.length ) return false; clearAttributes(); tokens[p++].copyTo(reusableToken); return true; } @Override public void reset() throws IOException { super.reset(); this.p = 0; } }; return new TokenStreamComponents(ts); }
public void test() throws IOException { Token token = new Token(); SingleTokenTokenStream ts = new SingleTokenTokenStream(token); AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(CharTermAttribute.class); assertTrue(tokenAtt instanceof Token); ts.reset(); assertTrue(ts.incrementToken()); assertEquals(token, tokenAtt); assertFalse(ts.incrementToken()); token = new Token("hallo", 10, 20, "someType"); ts.setToken(token); ts.reset(); assertTrue(ts.incrementToken()); assertEquals(token, tokenAtt); assertFalse(ts.incrementToken()); }
@Override public void copyTo(AttributeImpl target) { final List<CompiledAutomaton> targetAutomata = ((LevenshteinAutomataAttribute) target).automata(); targetAutomata.clear(); targetAutomata.addAll(automata); }
/** * Adds a {@link SinkTokenStream} created by another <code>TeeSinkTokenFilter</code> * to this one. The supplied stream will also receive all consumed tokens. * This method can be used to pass tokens from two different tees to one sink. */ public void addSinkTokenStream(final SinkTokenStream sink) { // check that sink has correct factory if (!this.getAttributeFactory().equals(sink.getAttributeFactory())) { throw new IllegalArgumentException("The supplied sink is not compatible to this tee"); } // add eventually missing attribute impls to the existing sink for (Iterator<AttributeImpl> it = this.cloneAttributes().getAttributeImplsIterator(); it.hasNext(); ) { sink.addAttributeImpl(it.next()); } this.sinks.add(new WeakReference<>(sink)); }
/** Make this tokenizer get attributes from the delegate token stream. */ private static final AttributeFactory delegatingAttributeFactory(final AttributeSource source) { return new AttributeFactory() { @Override public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) { return (AttributeImpl) source.addAttribute(attClass); } }; }
@Override public void copyTo(AttributeImpl target) { AdditionalTermAttribute t = (AdditionalTermAttribute) target; for(int inx=0;inx < additionalTerms.size(); inx++) { String term = additionalTerms.get(inx); String type = types.get(inx); @SuppressWarnings("rawtypes") List synonyms = this.synonyms; int[] offset = offsets.get(inx); int subLength = this.subLength; t.addAdditionalTerm(term, type, synonyms, subLength, offset[0], offset[1]); } }
@Override public void copyTo(AttributeImpl target) { List<StringBuilder> cloned = null; if (tags != null) { cloned = new ArrayList<>(tags.size()); for (StringBuilder b : tags) { cloned.add(new StringBuilder(b)); } } ((MorphosyntacticTagsAttribute) target).setTags(cloned); }
@Override public void copyTo(AttributeImpl target) { if (!(target instanceof UniqueFieldAttributeImpl)) { throw new IllegalArgumentException( "cannot copy the values from attribute UniqueFieldAttribute to an instance of " + target.getClass().getName()); } UniqueFieldAttributeImpl uniqueFieldAttr = (UniqueFieldAttributeImpl) target; uniqueFieldAttr.uniqueField = uniqueField.toString(); }
@Override public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) { if (attClass == TermToBytesRefAttribute.class) return new MyTermAttributeImpl(); if (CharTermAttribute.class.isAssignableFrom(attClass)) throw new IllegalArgumentException("no"); return delegate.createAttributeInstance(attClass); }
public static <T extends AttributeImpl> T assertCloneIsEqual(T att) { @SuppressWarnings("unchecked") T clone = (T) att.clone(); assertEquals("Clone must be equal", att, clone); assertEquals("Clone's hashcode must be equal", att.hashCode(), clone.hashCode()); return clone; }
public static <T extends AttributeImpl> T assertCopyIsEqual(T att) throws Exception { @SuppressWarnings("unchecked") T copy = (T) att.getClass().newInstance(); att.copyTo(copy); assertEquals("Copied instance must be equal", att, copy); assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode()); return copy; }
/** Make this Tokenizer get attributes from the delegate token stream. */ private static final AttributeFactory delegatingAttributeFactory(final AttributeSource source) { return new AttributeFactory() { @Override public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) { return (AttributeImpl) source.addAttribute(attClass); } }; }
private OffsetAttribute getOffsetAttribute( ) { // get char term attr from current state Iterator<AttributeImpl> attrIt = getAttributeImplsIterator(); while (attrIt != null && attrIt.hasNext() ) { AttributeImpl attrImp = attrIt.next(); if (attrImp instanceof OffsetAttribute) { return (OffsetAttribute)attrImp; } } return null; }
private PositionIncrementAttribute getPositionIncrementAttribute( ) { // get char term attr from current state Iterator<AttributeImpl> attrIt = getAttributeImplsIterator(); while (attrIt != null && attrIt.hasNext() ) { AttributeImpl attrImp = attrIt.next(); if (attrImp instanceof PositionIncrementAttribute) { return (PositionIncrementAttribute)attrImp; } } return null; }
@Override public AttributeImpl createAttributeInstance( Class<? extends Attribute> attClass) { return attClass.isAssignableFrom(ICUCollatedTermAttributeImpl.class) ? new ICUCollatedTermAttributeImpl(collator) : delegate.createAttributeInstance(attClass); }
@Override public void copyTo(AttributeImpl target) { List<StringBuilder> cloned = null; if (tags != null) { cloned = new ArrayList<StringBuilder>(tags.size()); for (StringBuilder b : tags) { cloned.add(new StringBuilder(b)); } } ((MorphosyntacticTagsAttribute) target).setTags(cloned); }
@Override public AttributeImpl createAttributeInstance( Class<? extends Attribute> attClass) { return attClass.isAssignableFrom(CollatedTermAttributeImpl.class) ? new CollatedTermAttributeImpl(collator) : delegate.createAttributeInstance(attClass); }
@Override public AttributeImpl createAttributeInstance( Class<? extends Attribute> attClass) { return attClass.isAssignableFrom(MockUTF16TermAttributeImpl.class) ? new MockUTF16TermAttributeImpl() : delegate.createAttributeInstance(attClass); }
private CharTermAttribute getTermAttribute() { Iterator<AttributeImpl> attrIt = getAttributeImplsIterator(); while (attrIt != null && attrIt.hasNext()) { AttributeImpl attrImp = attrIt.next(); if (attrImp instanceof CharTermAttribute) { return (CharTermAttribute) attrImp; } } return null; }
private OffsetAttribute getOffsetAttribute() { Iterator<AttributeImpl> attrIt = getAttributeImplsIterator(); while (attrIt != null && attrIt.hasNext()) { AttributeImpl attrImp = attrIt.next(); if (attrImp instanceof OffsetAttribute) { return (OffsetAttribute) attrImp; } } return null; }
private PositionIncrementAttribute getPositionIncrementAttribute() { Iterator<AttributeImpl> attrIt = getAttributeImplsIterator(); while (attrIt != null && attrIt.hasNext()) { AttributeImpl attrImp = attrIt.next(); if (attrImp instanceof PositionIncrementAttribute) { return (PositionIncrementAttribute) attrImp; } } return null; }
/** * Adds a {@link SinkTokenStream} created by another <code>TeeSinkTokenFilter</code> * to this one. The supplied stream will also receive all consumed tokens. * This method can be used to pass tokens from two different tees to one sink. */ public void addSinkTokenStream(final SinkTokenStream sink) { // check that sink has correct factory if (!this.getAttributeFactory().equals(sink.getAttributeFactory())) { throw new IllegalArgumentException("The supplied sink is not compatible to this tee"); } // add eventually missing attribute impls to the existing sink for (Iterator<AttributeImpl> it = this.cloneAttributes().getAttributeImplsIterator(); it.hasNext(); ) { sink.addAttributeImpl(it.next()); } this.sinks.add(new WeakReference<SinkTokenStream>(sink)); }
