Java 类org.eclipse.jface.text.rules.IToken 实例源码

@Override
public IToken nextToken() {
    if (currentTemplateTextToken != null) {
        if (currentTemplateTextToken.hasNext())
            return currentTemplateTextToken.nextToken();
        else
            currentTemplateTextToken = null;
    }
    if (!getIterator().hasNext())
        return Token.EOF;
    ILexerTokenRegion next = getIterator().next();
    int tokenType = next.getLexerTokenType();
    switch (tokenType) {
    case RULE_TEMPLATE_HEAD:
    case RULE_TEMPLATE_MIDDLE: {
        currentTemplateTextToken = createTemplateToken(tokenType, next);
        return currentTemplateTextToken.nextToken();
    }
    default:
        setCurrentToken(next);
        return createToken(next);
    }
}

private void buildWordRules(List<IPredicateRule> rules, IToken token, DocumentKeyWord[] values) {
    for (DocumentKeyWord keyWord : values) {
        ExactWordPatternRule rule1 = new ExactWordPatternRule(onlyLettersWordDetector, createWordStart(keyWord), token,
                keyWord.isBreakingOnEof());
        rule1.setAllowedPrefix('@');
        rule1.setAllowedPostfix(':');
        rules.add(rule1);

        ExactWordPatternRule rule2 = new ExactWordPatternRule(onlyLettersWordDetector, keyWord.getText().toUpperCase(), token,
                keyWord.isBreakingOnEof());
        rule2.setAllowedPrefix('@');
        rule2.setAllowedPostfix(':');
        rules.add(rule2);

    }
}

@Override
public IToken evaluate(ICharacterScanner scanner1, boolean resume) {
    if (!(scanner1 instanceof BatchDocumentPartitionScanner)){
        return Token.UNDEFINED;
    }
    BatchDocumentPartitionScanner scanner = (BatchDocumentPartitionScanner)scanner1;
    if (scanner.getOffset()<ECHO.length()){
        /* would be a problem on start on going back...*/
        return Token.UNDEFINED;
    }
    scanner.unread(); // go to character before
    int charBeforeAsInt = scanner.read(); // get character, moves to origin pos again
    /* fast guard closing:*/
    if (charBeforeAsInt!='o' && charBeforeAsInt!='O'){ // echo ECHO <- last character must be an o other wise guard close...
        return Token.UNDEFINED;
    }
    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (parserSupport.isAfterEchoHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    int r = scanner.read();
    scanner.unread();
    /* fast guard closing:*/
    if (ICharacterScanner.EOF==r || r!='<'){
        return Token.UNDEFINED;
    }

    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (hereStringSupport.isHereStringStateHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    int r = scanner.read();
    scanner.unread();
    /* fast guard closing:*/
    if (ICharacterScanner.EOF==r || r!='<'){
        return Token.UNDEFINED;
    }

    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (hereDocSupport.isHereDocStateHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

public RelationModelPartitionScanner() {
  final IToken commentPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_COMMENT);
  final IToken optionPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_OPTION);
  final IToken universePartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_UNIVERSE);
  final IToken relBoundPartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_REL_BOUND);

  final List<IRule> rules = new ArrayList<IRule>();
  rules.add(new MultiLineRule("/**", "**/", commentPartition));
  rules.add(new EndOfLineRule("--", commentPartition));
  // rules.add(new EndOfLineRule("//", commentPartition));
  rules.add(new MultiLineRule("options {", "}", optionPartition));
  rules.add(new MultiLineRule("universe {", "}", universePartition));
  rules.add(new MultiLineRule("universe [", "]", universePartition));
  rules.add(new RelationBoundsRule("relations", "}", relBoundPartition));

  final IPredicateRule[] result = new IPredicateRule[rules.size()];
  rules.toArray(result);
  this.setPredicateRules(result);
}

@Override
protected IToken doEvaluate(final ICharacterScanner scanner, final boolean resume) {

  if (resume) {
    if (this.endSequenceDetected(scanner)) {
      return this.fToken;
    }
  } else {
    final int c = scanner.read();
    if (c == this.fStartSequence[0]) {
      if (this.sequenceDetected(scanner, this.fStartSequence, false)) {
        if (this.endSequenceDetected(scanner)) {
          return this.fToken;
        }
      }
    }
  }

  scanner.unread();
  return Token.UNDEFINED;
}

public ModelScanner(final ColorManager colorManager) {

    final IToken bracketsToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.BRACKETS)));
    final IToken commToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.COMMENT)));

    final List<IRule> rules = new ArrayList<>();

    rules.add(new WhitespaceRule(new ValidationWhitespaceDetector()));
    rules.add(new BracketsRule(bracketsToken));
    rules.add(new MultiLineRule("/**", "**/", commToken));
    rules.add(new EndOfLineRule("--", commToken));

    final IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    this.setRules(result);
  }

public IToken evaluate(ICharacterScanner scanner) {

        buffer.setLength(0);

        int c = read(scanner);
        if (c == '&') {

            int i = 0;
            do {
                c = read(scanner);
                i++;

                if (c == '<' || c == ']') {
                    for (int j = i - 1; j > 0; j--)
                        scanner.unread();
                    return Token.UNDEFINED;
                }
            } while (c != ';');
            return fToken;
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

protected void addGroovyRules(List<IPredicateRule> rules) {
    IToken groovyAnnotation = createToken(ANNOTATION);
    IToken javaDocComment = createToken(GROOVY_DOC);
    IToken groovyComment = createToken(COMMENT);
    IToken groovySimpleString = createToken(STRING);
    IToken groovyGString = createToken(GSTRING);
    IToken groovyKeyWord = createToken(GROOVY_KEYWORD);
    IToken javaKeyWord = createToken(JAVA_KEYWORD);
    IToken javaLiteral = createToken(JAVA_LITERAL);

    rules.add(new MultiLineRule("/**", "*/", javaDocComment));
    rules.add(new MultiLineRule("/*", "*/", groovyComment));
    rules.add(new SingleLineRule("//", "", groovyComment));

    rules.add(new MultiLineRule("\"", "\"", groovyGString, '\\'));
    rules.add(new MultiLineRule("\'", "\'", groovySimpleString, '\\'));

    buildWordRules(rules, javaKeyWord, JavaKeyWords.values(), javaWordDetector);
    buildWordRules(rules, javaLiteral, JavaLiteralKeyWords.values(), javaWordDetector);

    buildWordRules(rules, groovyKeyWord, GroovyKeyWords.values(), javaWordDetector);

    buildAnnotationRules(rules, groovyAnnotation, onlyAnnotationWordDetector);
}

public EiffelPartitionScanner() {
    super();

    IToken multilineComment = new Token(MULTILINE_COMMENT);
    IToken singlelineComment = new Token(SINGLELINE_COMMENT);
    IToken string = new Token(STRING);

    List<Object> rules = new ArrayList<>();

    //Add rule for single line comments.
    rules.add(new EndOfLineRule("--", singlelineComment));

    // Add rule for strings and character constants.
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));
    rules.add(new SingleLineRule("'", "'", string, '\\'));

    //Add rules for multi-line comments and eiffeldocs
    rules.add(new MultiLineRule("--[[", "]]", multilineComment, (char) 0, true));

    IPredicateRule[] result = new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

@Override
protected List<IRule> createRules() {
    IToken keywordToken = getToken(Prefs.EDITOR_MATH_KEYWORD_COLOR);
    IToken symbolToken = getToken(Prefs.EDITOR_MATH_SYMBOL_COLOR);
    IToken commentToken = getToken(Prefs.EDITOR_MATH_COMMENT_COLOR);
    // IToken defaultToken = getToken(Prefs.EDITOR_DEFAULT_COLOR);

    WordRule wordRule = new WordRule(new MathWordDetector(), keywordToken, true);
    WordRule symbolRule = new WordRule(new MathSymbolDetector(), symbolToken, true);

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(wordRule);
    rules.add(symbolRule);
    rules.add(new EndOfLineRule("%", commentToken));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    return rules;
}

protected de.darwinspl.preferences.resource.dwprofile.ui.IDwprofileTokenScanner scan(EObject object, ITokenScannerConstraint constraint) {
    IDocument document = getSourceViewer().getDocument();

    de.darwinspl.preferences.resource.dwprofile.IDwprofileLocationMap locationMap = textResource.getLocationMap();

    de.darwinspl.preferences.resource.dwprofile.ui.IDwprofileTokenScanner tokenScanner = createTokenScanner();
    int offset = locationMap.getCharStart(object);
    int length = getLength(locationMap, object);

    tokenScanner.setRange(document, offset, length);
    IToken token = tokenScanner.nextToken();
    while (!token.isEOF()) {
        if (constraint.mustStop(tokenScanner)) {
            return tokenScanner;
        }
        token = tokenScanner.nextToken();
    }
    return null;
}

protected eu.hyvar.context.contextValidity.resource.hyvalidityformula.ui.IHyvalidityformulaTokenScanner scan(EObject object, ITokenScannerConstraint constraint) {
    IDocument document = getSourceViewer().getDocument();

    eu.hyvar.context.contextValidity.resource.hyvalidityformula.IHyvalidityformulaLocationMap locationMap = textResource.getLocationMap();

    eu.hyvar.context.contextValidity.resource.hyvalidityformula.ui.IHyvalidityformulaTokenScanner tokenScanner = createTokenScanner();
    int offset = locationMap.getCharStart(object);
    int length = getLength(locationMap, object);

    tokenScanner.setRange(document, offset, length);
    IToken token = tokenScanner.nextToken();
    while (!token.isEOF()) {
        if (constraint.mustStop(tokenScanner)) {
            return tokenScanner;
        }
        token = tokenScanner.nextToken();
    }
    return null;
}

public XMLTagScanner(ColorManager manager) {
    IToken string = new Token(new TextAttribute(manager
            .getColor(IXMLColorConstants.STRING)));

    IToken attrValue = new Token(XML_ATTR_VALUE);

    IRule[] rules = new IRule[3];

    // Add rule for double quotes
    rules[0] = new SingleLineRule("\"", "\"", string, '\\');
    // Add a rule for single quotes
    rules[1] = new SingleLineRule("'", "'", string, '\\');
    // Add generic whitespace rule.
    rules[2] = new WhitespaceRule(new NCLWhitespaceDetector());

    setRules(rules);
}

protected eu.hyvar.mspl.manifest.resource.hymanifest.ui.IHymanifestTokenScanner scan(EObject object, ITokenScannerConstraint constraint) {
    IDocument document = getSourceViewer().getDocument();

    eu.hyvar.mspl.manifest.resource.hymanifest.IHymanifestLocationMap locationMap = textResource.getLocationMap();

    eu.hyvar.mspl.manifest.resource.hymanifest.ui.IHymanifestTokenScanner tokenScanner = createTokenScanner();
    int offset = locationMap.getCharStart(object);
    int length = getLength(locationMap, object);

    tokenScanner.setRange(document, offset, length);
    IToken token = tokenScanner.nextToken();
    while (!token.isEOF()) {
        if (constraint.mustStop(tokenScanner)) {
            return tokenScanner;
        }
        token = tokenScanner.nextToken();
    }
    return null;
}

public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      int c = read(scanner);
      if (c == '&') {

         int i = 0;
         do {
            c = read(scanner);
            i++;

            if (c == '<' || c == ']') {
               for (int j = i - 1; j > 0; j--)
                  scanner.unread();
               return Token.UNDEFINED;
            }
         } while (c != ';');
         return fToken;
      }

      scanner.unread();
      return Token.UNDEFINED;
   }

private SqlCommentPartitionScanner() {
    IToken commentToken = new Token(SQL_COMMENT);

    List<IPredicateRule> rules= new ArrayList<>();

    rules.add(new EndOfLineRule("--", commentToken, '\\'));
    rules.add(new MultiLineRule("/*", "*/", commentToken, '\\'));

    rules.add(new WordPredicateRule(commentToken));
    rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\')); 
    rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\'));

    IPredicateRule[] result= new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

public PartitionScanner() {
    super();

    IToken matter = new Token(Partitions.FRONT_MATTER);
    IToken comment = new Token(Partitions.COMMENT);
    IToken codeblock = new Token(Partitions.CODEBLOCK);
    IToken htmlblock = new Token(Partitions.HTMLBLOCK);
    IToken dotblock = new Token(Partitions.DOTBLOCK);
    IToken mathblock = new Token(Partitions.MATHBLOCK);

    List<IRule> rules = new ArrayList<IRule>();

    rules.add(new FrontMatterRule("---", "---", matter, '\\'));
    rules.add(new MatchRule("\\$\\S", "\\S\\$", mathblock, '\\', false));
    rules.add(new MultiLineRule("$$", "$$", mathblock, '\\'));
    rules.add(new MultiLineRule("<!--", "-->", comment, '\\'));
    rules.add(new HtmlCodeRule(htmlblock));
    rules.add(new DotCodeRule(dotblock));
    rules.add(new MultiLineRule("~~~", "~~~", codeblock, '\\'));
    rules.add(new MultiLineRule("```", "```", codeblock, '\\'));
    rules.add(new IndentedCodeRule(codeblock));

    IPredicateRule[] rule = new IPredicateRule[rules.size()];
    setPredicateRules(rules.toArray(rule));
}

/**
 * Does the actual evaluation of the stream.
 * 
 * @param scanner The scanner
 * @param count The initial count of {
 * @return <code>fToken</code> on success, <code>Token.UNDEFINED</code> if
 * the match doesn't succeed
 */
private IToken doEvaluate(ICharacterScanner scanner, int count) {
    boolean inString = false;
    int c = scanner.read();

    if (((char) c) == '{') {
        do {
            c = scanner.read();
            if (((char) c) == '{' && !inString)
                count++;
            else if (((char) c) == '}' && !inString)
                count--;
            else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (count > 0);
        return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}

@Override
protected List<IRule> createRules() {
    IToken keywordToken = getToken(Prefs.EDITOR_HTML_KEYWORD_COLOR);
    IToken symbolToken = getToken(Prefs.EDITOR_HTML_SYMBOL_COLOR);
    IToken stringToken = getToken(Prefs.EDITOR_HTML_STRING_COLOR);
    IToken defaultToken = getToken(Prefs.EDITOR_DEFAULT_COLOR);

    WordRule wordRule = new WordRule(new HtmlWordDetector(), defaultToken, true);
    for (String keyword : keywords) {
        wordRule.addWord(keyword, keywordToken);
    }

    WordRule symbolRule = new WordRule(new HtmlSymbolDetector(), defaultToken, true);
    for (String symbol : symbols) {
        symbolRule.addWord(symbol, symbolToken);
    }

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new SingleLineRule("\"", "\"", stringToken, '\\', true));
    rules.add(new SingleLineRule("'", "'", stringToken, '\\', true));
    rules.add(wordRule);
    rules.add(symbolRule);
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    return rules;
}

@Override
public IToken evaluate(ICharacterScanner scanner) {
    found = false;

    int c = scanner.read();
    scanner.unread();
    if (isHws(c)) {
        int col = scanner.getColumn();
        if (col == 1) {
            scanner.unread();
            int p = scanner.read();
            if (p == '\n') col = 0;
        }
        if (col == 0) {
            tabWidth = FluentMkUI.getDefault().getPreferenceStore().getInt(Prefs.EDITOR_TAB_WIDTH);
            if (tabWidth < 2) tabWidth = 4;
            line = 0;
            while (evaluateLine(scanner)) {
                found = true;
                line++;
            }
        }
    }
    if (found) return token;
    return Token.UNDEFINED;
}

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public BibPartitionScanner() {
    super();

    IToken bibEntry = new Token(BIB_ENTRY);

    List rules = new ArrayList();

    // Add rule for single line comments.
    // rules.add(new EndOfLineRule("//", Token.UNDEFINED));

    // Add rule for strings and character constants.
    // rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\'));
    // rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\'));

    // Add rules for BibTeX entries
    //rules.add(new MultiLineRule("{", "}", bibEntry, (char) 0, false));
    rules.add(new BibBraceRule(true, bibEntry));;

    IPredicateRule[] result= new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

public IToken evaluate(ICharacterScanner scanner) {

        buffer.setLength(0);

        charsRead = 0;
        int c = read(scanner);

        if (c == matchString.charAt(0)) {
            do {
                c = read(scanner);
            } while (isOK((char) c));

            if (charsRead == matchString.length()) {
                return fToken;
            } else {
                rewind(scanner);
                return Token.UNDEFINED;
            }

        }

        scanner.unread();
        return Token.UNDEFINED;
    }

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

/**
 * A default constructor.
 * @param manager
 */
public TexOptArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

public static WordRule buildRule(ImpexRules ruleType, IToken token) {
    WordRule result = null;
    switch (ruleType) {
    case KEYWORD:
        result = buildKeywordRule(token);
        break;
    case VARIABLE:
        result = buildVariableRule('$', token);
        break;
    case REFERENCE:
        result = buildVariableRule('&', token);
        break;
    case SEMICOLON:
        result = buildCharRule(';', token);
        break;
    case COMMA:
        result = buildCharRule(',', token);
        break;
    default:
        result = null;
        break;
    }
    return result;
}

protected IToken createToken() {
    String attributeId = delimiter
            ? HighlightingConfiguration.TEMPLATE_DELIMITER_ID
            : HighlightingConfiguration.TEMPLATE_ID;
    Token token = new Token(getAttribute(attributeId));
    return token;
}

@Override
protected void addOtherRules(List<IPredicateRule> rules) {
    IToken jenkinsDefaultClosureKeywords = createToken(JENKINS_KEYWORD);
    IToken jenkinsVariables = createToken(JENKINS_VARIABLE);

    buildWordRules(rules, jenkinsDefaultClosureKeywords, JenkinsDefaultClosureKeyWords.values(),onlyLettersWordDetector);
    buildWordRules(rules, jenkinsVariables, JenkinsSpecialVariableKeyWords.values(),onlyLettersWordDetector);
}

public BatchVariableRule(IToken token) {
    this.token = token;
    notValidCharacterSet=new HashSet<>();
    for (char c: NOT_VALID_CHARACTERS){
        /* transform to collection with not primitive by auto boxing:*/
        notValidCharacterSet.add(c);
    }
}

public BatchDocumentPartitionScanner() {
        IToken echoOutput = createToken(ECHO_OUTPUT);
        IToken parameters = createToken(PARAMETER);
        IToken comment = createToken(COMMENT);
        IToken doubleString = createToken(DOUBLE_STRING);

        IToken batchBuildIn = createToken(BATCH_KEYWORD);

        IToken knownVariables = createToken(KNOWN_VARIABLES);
        IToken variables = createToken(VARIABLES);
        IToken batchExternalCommands = createToken(BATCH_COMMAND);

        List<IPredicateRule> rules = new ArrayList<>();
        rules.add(new AfterEchoDocumentRule(echoOutput)); 
        rules.add(new BatchVariableRule(variables));
        rules.add(new SingleLineRule("rem", "", comment, (char) -1, true));
        rules.add(new SingleLineRule("REM", "", comment, (char) -1, true));
        rules.add(new SingleLineRule("::", "", comment, (char) -1, true));

        rules.add(new BatchStringRule("\"", "\"", doubleString));

        rules.add(new CommandParameterRule(parameters));

        buildWordRules(rules, batchBuildIn, BatchBuildInKeywords.values());
        buildWordRules(rules, batchExternalCommands, BatchExternalKeyWords.values());

        buildWordRules(rules, knownVariables, BatchSpecialVariableKeyWords.values());

        /* TODO ATR: 24.11.2017: remove the next linecomplete or support variable setup like in bash editor*/
//      buildVarDefRules(rules, knownVariables, BatchSpecialVariableKeyWords.values());

        setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
    }

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume) {

        if (resume) {

            if (endSequenceDetected(scanner))
                return fToken;

        } else {
            int c= scanner.read();
            if (c == fStartSequence[0]) {
                scanner.unread();
                scanner.unread();
                int before = scanner.read();

                if (before=='\\'){
                    return Token.UNDEFINED;
                }
                scanner.read();

                if (sequenceDetected(scanner, fStartSequence, false)) {
                    if (endSequenceDetected(scanner))
                        return fToken;
                }
            }
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    char start = (char) scanner.read();
    if ('-' !=start) {
        scanner.unread();
        return Token.UNDEFINED;
    }

    /* check if the former character is a space - if not this is not an argument */
    scanner.unread();
    scanner.unread();
    char beforeStart = (char) scanner.read();
    if (' ' !=beforeStart){
        /* no unread necessary, scanner is at start start*/
        return Token.UNDEFINED;
    }
    /* go after start again*/
    scanner.read();

    /* okay could be a parameter*/
    do {
        char c = (char) scanner.read();
        if (! isPartOfParameter(c)) {
            scanner.unread();
            break;
        }
    } while (true);
    return getSuccessToken();
}

@Test
public void function_with_keyword_help_inside_is_found_as_funtion() {
    scanner = new OneLineSimpleTestCharacterScanner("function show_help_and_exit(){}");
    ExactWordPatternRule rule = new ExactWordPatternRule(detector, "function", token);

    /* execute */
    IToken result = rule.evaluate(scanner);

    /* test */
    assertEquals(token, result);
    assertEquals(8,scanner.column);

}

public XMLTagScanner(ColorManager manager) {
    IToken string = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.STRING)));

    IRule[] rules = new IRule[3];

    // Add rule for double quotes
    rules[0] = new SingleLineRule("\"", "\"", string, '\\');
    // Add a rule for single quotes
    rules[1] = new SingleLineRule("'", "'", string, '\\');
    // Add generic whitespace rule.
    rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

    setRules(rules);
}

public BashDocumentPartitionScanner() {
        IToken hereDocument = createToken(HERE_DOCUMENT);
        IToken hereString = createToken(HERE_STRING);
        IToken parameters = createToken(PARAMETER);
        IToken comment = createToken(COMMENT);
        IToken simpleString = createToken(SINGLE_STRING);
        IToken doubleString = createToken(DOUBLE_STRING);
        IToken backtickString = createToken(BACKTICK_STRING);

        IToken systemKeyword = createToken(BASH_SYSTEM_KEYWORD);
        IToken bashKeyword = createToken(BASH_KEYWORD);

        IToken knownVariables = createToken(KNOWN_VARIABLES);
        IToken variables = createToken(VARIABLES);
        IToken includeKeyword = createToken(INCLUDE_KEYWORD);
        IToken bashCommand = createToken(BASH_COMMAND);

        List<IPredicateRule> rules = new ArrayList<>();
        rules.add(new HereStringRule(hereString));
        rules.add(new HereDocumentRule(hereDocument));

        buildWordRules(rules, systemKeyword, BashSystemKeyWords.values());
        rules.add(new BashVariableRule(variables));
        rules.add(new SingleLineRule("#", "", comment, (char) -1, true));

        rules.add(new BashStringRule("\"", "\"", doubleString));
        rules.add(new BashStringRule("\'", "\'", simpleString));
        rules.add(new BashStringRule("`", "`", backtickString));

        rules.add(new CommandParameterRule(parameters));

        buildWordRules(rules, includeKeyword, BashIncludeKeyWords.values());
        buildWordRules(rules, bashKeyword, BashLanguageKeyWords.values());
        buildWordRules(rules, bashCommand, BashGnuCommandKeyWords.values());

//      buildVarDefRules(rules, knownVariables, BashSpecialVariableKeyWords.values());
        buildWordRules(rules, knownVariables, BashSpecialVariableKeyWords.values());

        setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
    }

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume) {

        if (resume) {

            if (endSequenceDetected(scanner))
                return fToken;

        } else {
            int c= scanner.read();
            if (c == fStartSequence[0]) {
                scanner.unread();
                scanner.unread();
                int before = scanner.read();

                if (before=='\\'){
                    return Token.UNDEFINED;
                }
                scanner.read();

                if (sequenceDetected(scanner, fStartSequence, false)) {
                    if (endSequenceDetected(scanner))
                        return fToken;
                }
            }
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    char start = (char) scanner.read();
    if ('-' !=start) {
        scanner.unread();
        return Token.UNDEFINED;
    }

    /* check if the former character is a space - if not this is not an argument */
    scanner.unread();
    scanner.unread();
    char beforeStart = (char) scanner.read();
    if (' ' !=beforeStart){
        /* no unread necessary, scanner is at start start*/
        return Token.UNDEFINED;
    }
    /* go after start again*/
    scanner.read();

    /* okay could be a parameter*/
    do {
        char c = (char) scanner.read();
        if (! isPartOfParameter(c)) {
            scanner.unread();
            break;
        }
    } while (true);
    return getSuccessToken();
}

@Test
public void line_starting_with_delimiter_returns_token() {
    /* prepare */
    scanner = new SimpleTestCharacterScanner("<< EOF\nsomething\nEOF");
    rule.trace=true;

    /* execute */
    IToken result = rule.evaluate(scanner);

    /* test */
    assertEquals(token,result);

}

@Test
public void line_starting_with_space_followed_by_delimiter_returns_not_token_but_undefined() {
    /* prepare */
    scanner = new SimpleTestCharacterScanner(" << EOF\nsomething\nEOF");
    rule.trace=true;

    /* execute */
    IToken result = rule.evaluate(scanner);

    /* test */
    assertEquals(Token.UNDEFINED,result);

}