Java 类org.eclipse.jface.text.rules.Token 实例源码

@Override
public IToken nextToken() {
    if (currentTemplateTextToken != null) {
        if (currentTemplateTextToken.hasNext())
            return currentTemplateTextToken.nextToken();
        else
            currentTemplateTextToken = null;
    }
    if (!getIterator().hasNext())
        return Token.EOF;
    ILexerTokenRegion next = getIterator().next();
    int tokenType = next.getLexerTokenType();
    switch (tokenType) {
    case RULE_TEMPLATE_HEAD:
    case RULE_TEMPLATE_MIDDLE: {
        currentTemplateTextToken = createTemplateToken(tokenType, next);
        return currentTemplateTextToken.nextToken();
    }
    default:
        setCurrentToken(next);
        return createToken(next);
    }
}

@Override
public IToken evaluate(ICharacterScanner scanner1, boolean resume) {
    if (!(scanner1 instanceof BatchDocumentPartitionScanner)){
        return Token.UNDEFINED;
    }
    BatchDocumentPartitionScanner scanner = (BatchDocumentPartitionScanner)scanner1;
    if (scanner.getOffset()<ECHO.length()){
        /* would be a problem on start on going back...*/
        return Token.UNDEFINED;
    }
    scanner.unread(); // go to character before
    int charBeforeAsInt = scanner.read(); // get character, moves to origin pos again
    /* fast guard closing:*/
    if (charBeforeAsInt!='o' && charBeforeAsInt!='O'){ // echo ECHO <- last character must be an o other wise guard close...
        return Token.UNDEFINED;
    }
    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (parserSupport.isAfterEchoHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    int r = scanner.read();
    scanner.unread();
    /* fast guard closing:*/
    if (ICharacterScanner.EOF==r || r!='<'){
        return Token.UNDEFINED;
    }

    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (hereStringSupport.isHereStringStateHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    int r = scanner.read();
    scanner.unread();
    /* fast guard closing:*/
    if (ICharacterScanner.EOF==r || r!='<'){
        return Token.UNDEFINED;
    }

    ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
    if (hereDocSupport.isHereDocStateHandled(codePosSupport)){
        return getSuccessToken();
    }
    codePosSupport.resetToStartPos();
    return Token.UNDEFINED;
}

public RelationModelPartitionScanner() {
  final IToken commentPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_COMMENT);
  final IToken optionPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_OPTION);
  final IToken universePartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_UNIVERSE);
  final IToken relBoundPartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_REL_BOUND);

  final List<IRule> rules = new ArrayList<IRule>();
  rules.add(new MultiLineRule("/**", "**/", commentPartition));
  rules.add(new EndOfLineRule("--", commentPartition));
  // rules.add(new EndOfLineRule("//", commentPartition));
  rules.add(new MultiLineRule("options {", "}", optionPartition));
  rules.add(new MultiLineRule("universe {", "}", universePartition));
  rules.add(new MultiLineRule("universe [", "]", universePartition));
  rules.add(new RelationBoundsRule("relations", "}", relBoundPartition));

  final IPredicateRule[] result = new IPredicateRule[rules.size()];
  rules.toArray(result);
  this.setPredicateRules(result);
}

@Override
protected IToken doEvaluate(final ICharacterScanner scanner, final boolean resume) {

  if (resume) {
    if (this.endSequenceDetected(scanner)) {
      return this.fToken;
    }
  } else {
    final int c = scanner.read();
    if (c == this.fStartSequence[0]) {
      if (this.sequenceDetected(scanner, this.fStartSequence, false)) {
        if (this.endSequenceDetected(scanner)) {
          return this.fToken;
        }
      }
    }
  }

  scanner.unread();
  return Token.UNDEFINED;
}

public ModelScanner(final ColorManager colorManager) {

    final IToken bracketsToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.BRACKETS)));
    final IToken commToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.COMMENT)));

    final List<IRule> rules = new ArrayList<>();

    rules.add(new WhitespaceRule(new ValidationWhitespaceDetector()));
    rules.add(new BracketsRule(bracketsToken));
    rules.add(new MultiLineRule("/**", "**/", commToken));
    rules.add(new EndOfLineRule("--", commToken));

    final IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    this.setRules(result);
  }

public EiffelPartitionScanner() {
    super();

    IToken multilineComment = new Token(MULTILINE_COMMENT);
    IToken singlelineComment = new Token(SINGLELINE_COMMENT);
    IToken string = new Token(STRING);

    List<Object> rules = new ArrayList<>();

    //Add rule for single line comments.
    rules.add(new EndOfLineRule("--", singlelineComment));

    // Add rule for strings and character constants.
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));
    rules.add(new SingleLineRule("'", "'", string, '\\'));

    //Add rules for multi-line comments and eiffeldocs
    rules.add(new MultiLineRule("--[[", "]]", multilineComment, (char) 0, true));

    IPredicateRule[] result = new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

public XMLTextScanner( ColorManager colorManager) {

      ESCAPED_CHAR = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.ESCAPED_CHAR)));
      CDATA_START = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA)));
      CDATA_END = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA)));
      CDATA_TEXT = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA_TEXT)));
      IRule[] rules = new IRule[2];

      // Add rule to pick up escaped chars
      // Add rule to pick up start of CDATA section
      rules[0] = new CDataRule(CDATA_START, true);
      // Add a rule to pick up end of CDATA sections
      rules[1] = new CDataRule(CDATA_END, false);
      setRules(rules);

   }

public XMLPartitionScanner() {

      IToken xmlComment = new Token(XML_COMMENT);
      IToken xmlPI = new Token(XML_PI);
      IToken startTag = new Token(XML_START_TAG);
      IToken endTag = new Token(XML_END_TAG);
      IToken docType = new Token(XML_DOCTYPE);
      IToken text = new Token(XML_TEXT);

      IPredicateRule[] rules = new IPredicateRule[7];

      rules[0] = new NonMatchingRule();
      rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
      rules[2] = new MultiLineRule("<?", "?>", xmlPI);
      rules[3] = new MultiLineRule("</", ">", endTag);
      rules[4] = new StartTagRule(startTag);
      rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
      rules[6] = new XMLTextPredicateRule(text);

      setPredicateRules(rules);
   }

public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      int c = read(scanner);
      if (c == '&') {

         int i = 0;
         do {
            c = read(scanner);
            i++;

            if (c == '<' || c == ']') {
               for (int j = i - 1; j > 0; j--)
                  scanner.unread();
               return Token.UNDEFINED;
            }
         } while (c != ';');
         return fToken;
      }

      scanner.unread();
      return Token.UNDEFINED;
   }

public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      charsRead = 0;
      int c = read(scanner);

      if (c == matchString.charAt(0)) {
         do {
            c = read(scanner);
         } while (isOK((char) c));

         if (charsRead == matchString.length()) {
            return fToken;
         } else {
            rewind(scanner);
            return Token.UNDEFINED;
         }

      }

      scanner.unread();
      return Token.UNDEFINED;
   }

private SqlCommentPartitionScanner() {
    IToken commentToken = new Token(SQL_COMMENT);

    List<IPredicateRule> rules= new ArrayList<>();

    rules.add(new EndOfLineRule("--", commentToken, '\\'));
    rules.add(new MultiLineRule("/*", "*/", commentToken, '\\'));

    rules.add(new WordPredicateRule(commentToken));
    rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\')); 
    rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\'));

    IPredicateRule[] result= new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

/**
 * Does the actual evaluation of the stream.
 * 
 * @param scanner The scanner
 * @param count The initial count of {
 * @return <code>fToken</code> on success, <code>Token.UNDEFINED</code> if
 * the match doesn't succeed
 */
private IToken doEvaluate(ICharacterScanner scanner, int count) {
    boolean inString = false;
    int c = scanner.read();

    if (((char) c) == '{') {
        do {
            c = scanner.read();
            if (((char) c) == '{' && !inString)
                count++;
            else if (((char) c) == '}' && !inString)
                count--;
            else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (count > 0);
        return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}

public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (resume) {
        boolean inString = false;
        do {
            int c = scanner.read();
            if (((char) c) == ',' && !inString)
                break;
            else if (((char) c) == '@') {
                scanner.unread();
                return Token.UNDEFINED;
            } else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (true);
    }
    return doEvaluate(scanner, 1);
}

public IToken evaluate(ICharacterScanner scanner) {
    int count = 0;
    int charsRead = 0;
    int c = scanner.read();
    if (((char) c) == '"') {
            do {
                c = scanner.read();
                charsRead++;
                if (c == ICharacterScanner.EOF) {
                    unwind(scanner, charsRead);
                    return Token.UNDEFINED;
                } else if (((char) c) == '{') {
                    count++;
                } else if (((char) c) == '}') {
                    if (count == 0) {
                        unwind(scanner, charsRead);
                        return Token.UNDEFINED;
                    }
                    count--;
                }
            } while (((char) c) != '"');
            return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public BibPartitionScanner() {
    super();

    IToken bibEntry = new Token(BIB_ENTRY);

    List rules = new ArrayList();

    // Add rule for single line comments.
    // rules.add(new EndOfLineRule("//", Token.UNDEFINED));

    // Add rule for strings and character constants.
    // rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\'));
    // rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\'));

    // Add rules for BibTeX entries
    //rules.add(new MultiLineRule("{", "}", bibEntry, (char) 0, false));
    rules.add(new BibBraceRule(true, bibEntry));;

    IPredicateRule[] result= new IPredicateRule[rules.size()];
    rules.toArray(result);
    setPredicateRules(result);
}

@Override
public IToken evaluate(ICharacterScanner scanner) {
    found = false;

    int c = scanner.read();
    scanner.unread();
    if (isHws(c)) {
        int col = scanner.getColumn();
        if (col == 1) {
            scanner.unread();
            int p = scanner.read();
            if (p == '\n') col = 0;
        }
        if (col == 0) {
            tabWidth = FluentMkUI.getDefault().getPreferenceStore().getInt(Prefs.EDITOR_TAB_WIDTH);
            if (tabWidth < 2) tabWidth = 4;
            line = 0;
            while (evaluateLine(scanner)) {
                found = true;
                line++;
            }
        }
    }
    if (found) return token;
    return Token.UNDEFINED;
}

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

/**
 * A default constructor.
 * @param manager
 */
public TexOptArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

@Override
public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if (c != ICharacterScanner.EOF && 0 == scanner.getColumn() - 1) {
        try {
            if (dotStarts(scanner, c)) {
                gotoEOB(scanner, c);
                return getSuccessToken();
            }
        } catch (BadLocationException e) {
            Log.error("DotCodeRule", e);
        }
    }

    scanner.unread();
    return Token.UNDEFINED;
}

public XMLTextScanner(ColorManager colorManager) {

        ESCAPED_CHAR = new Token(new TextAttribute(colorManager
                .getColor(IXMLColorConstants.ESCAPED_CHAR)));
        CDATA_START = new Token(new TextAttribute(colorManager
                .getColor(IXMLColorConstants.CDATA)));
        CDATA_END = new Token(new TextAttribute(colorManager
                .getColor(IXMLColorConstants.CDATA)));
        CDATA_TEXT = new Token(new TextAttribute(colorManager
                .getColor(IXMLColorConstants.CDATA_TEXT)));
        IRule[] rules = new IRule[2];

        // Add rule to pick up escaped chars
        // Add rule to pick up start of CDATA section
        //rules[0] = new CDataRule(CDATA_START, true);
        // Add a rule to pick up end of CDATA sections
        //rules[1] = new CDataRule(CDATA_END, false);
        setRules(rules);

    }

public XMLPartitionScanner() {

        IToken xmlComment = new Token(XML_COMMENT);
        IToken xmlPI = new Token(XML_PI);
        IToken startTag = new Token(XML_START_TAG);
        IToken endTag = new Token(XML_END_TAG);
        IToken docType = new Token(XML_DOCTYPE);
        // IToken text = new Token(XML_TEXT);
        // IToken tag = new Token(XML_TAG);

        IPredicateRule[] rules = new IPredicateRule[6];

        rules[0] = new NonMatchingRule();
        rules[1] = new MultiLineRule("<!--", "-->", xmlComment, '\\', true);
        rules[2] = new MultiLineRule("<?", "?>", xmlPI);
        rules[3] = new MultiLineRule("</", ">", endTag);
        rules[4] = new StartTagRule(startTag);
        rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);

        // rules[6] = new XMLTextPredicateRule(text);
        // rules[7] = new TagRule(tag);

        setPredicateRules(rules);
    }

@Override
public IToken evaluate(ICharacterScanner scanner) {
    int col = scanner.getColumn();
    if (col == 1) {
        scanner.unread();
        int p = scanner.read();
        if (p == '\n') col = 0;
    }

    try {
        if (col == 0) {
            String text = getScanText(scanner);
            if (tagStarts(text)) {
                gotoEOB(scanner);
                return getSuccessToken();
            }
        }
    } catch (BadLocationException e) {
        Log.error("HtmlCodeRule", e);
    }
    return Token.UNDEFINED;
}

public IToken evaluate(ICharacterScanner scanner) {

        buffer.setLength(0);

        int c = read(scanner);
        if (c == '&') {

            int i = 0;
            do {
                c = read(scanner);
                i++;

                if (c == '<' || c == ']') {
                    for (int j = i - 1; j > 0; j--)
                        scanner.unread();
                    return Token.UNDEFINED;
                }
            } while (c != ';');
            return fToken;
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

public IToken evaluate(ICharacterScanner scanner) {

        buffer.setLength(0);

        charsRead = 0;
        int c = read(scanner);

        if (c == matchString.charAt(0)) {
            do {
                c = read(scanner);
            } while (isOK((char) c));

            if (charsRead == matchString.length()) {
                return fToken;
            } else {
                rewind(scanner);
                return Token.UNDEFINED;
            }

        }

        scanner.unread();
        return Token.UNDEFINED;
    }

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (fColumn == UNDEFINED) return doEvaluate(scanner, resume);

    int c = scanner.read();
    scanner.unread();
    if (c == fStartSequence[0]) {
        int col = scanner.getColumn();
        if (col == 1) {
            scanner.unread();
            int p = scanner.read();
            if (p == '\n') col = 0;
        }
        return (fColumn == col ? doEvaluate(scanner, resume) : Token.UNDEFINED);
    }
    return Token.UNDEFINED;
}

private void adaptToColorChange(Token token, PropertyChangeEvent event) {
    RGB rgb = null;

    Object value = event.getNewValue();
    if (value instanceof RGB) {
        rgb = (RGB) value;
    } else if (value instanceof String) {
        rgb = StringConverter.asRGB((String) value);
    }

    if (rgb != null) {

        String property = event.getProperty();
        Color color = colorMgr.getColor(property);
        if (!rgb.equals(color.getRGB())) {
            colorMgr.setValue(property, rgb);
            color = colorMgr.getColor(property);
        }

        Object data = token.getData();
        if (data instanceof TextAttribute) {
            TextAttribute oldAttr = (TextAttribute) data;
            token.setData(new TextAttribute(color, oldAttr.getBackground(), oldAttr.getStyle()));
        }
    }
}

private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
    boolean eventValue = false;
    Object value = event.getNewValue();
    if (value instanceof Boolean)
        eventValue = ((Boolean) value).booleanValue();
    else if (IPreferenceStore.TRUE.equals(value)) eventValue = true;

    Object data = token.getData();
    if (data instanceof TextAttribute) {
        TextAttribute oldAttr = (TextAttribute) data;
        boolean activeValue = (oldAttr.getStyle() & styleAttribute) == styleAttribute;
        if (activeValue != eventValue)
            token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(),
                    eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
    }
}

protected IToken createToken() {
    String attributeId = delimiter
            ? HighlightingConfiguration.TEMPLATE_DELIMITER_ID
            : HighlightingConfiguration.TEMPLATE_ID;
    Token token = new Token(getAttribute(attributeId));
    return token;
}

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume) {

        if (resume) {

            if (endSequenceDetected(scanner))
                return fToken;

        } else {
            int c= scanner.read();
            if (c == fStartSequence[0]) {
                scanner.unread();
                scanner.unread();
                int before = scanner.read();

                if (before=='\\'){
                    return Token.UNDEFINED;
                }
                scanner.read();

                if (sequenceDetected(scanner, fStartSequence, false)) {
                    if (endSequenceDetected(scanner))
                        return fToken;
                }
            }
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    char start = (char) scanner.read();
    if ('-' !=start) {
        scanner.unread();
        return Token.UNDEFINED;
    }

    /* check if the former character is a space - if not this is not an argument */
    scanner.unread();
    scanner.unread();
    char beforeStart = (char) scanner.read();
    if (' ' !=beforeStart){
        /* no unread necessary, scanner is at start start*/
        return Token.UNDEFINED;
    }
    /* go after start again*/
    scanner.read();

    /* okay could be a parameter*/
    do {
        char c = (char) scanner.read();
        if (! isPartOfParameter(c)) {
            scanner.unread();
            break;
        }
    } while (true);
    return getSuccessToken();
}

protected XMLScanner getXMLScanner() {
    if (scanner == null) {
        scanner = new XMLScanner(colorManager);
        scanner.setDefaultReturnToken(new Token(new TextAttribute(
                colorManager.getColor(IXMLColorConstants.DEFAULT))));
    }
    return scanner;
}

protected XMLTagScanner getXMLTagScanner() {
    if (tagScanner == null) {
        tagScanner = new XMLTagScanner(colorManager);
        tagScanner.setDefaultReturnToken(new Token(new TextAttribute(
                colorManager.getColor(IXMLColorConstants.TAG))));
    }
    return tagScanner;
}

public XMLTagScanner(ColorManager manager) {
    IToken string = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.STRING)));

    IRule[] rules = new IRule[3];

    // Add rule for double quotes
    rules[0] = new SingleLineRule("\"", "\"", string, '\\');
    // Add a rule for single quotes
    rules[1] = new SingleLineRule("'", "'", string, '\\');
    // Add generic whitespace rule.
    rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

    setRules(rules);
}

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume) {

        if (resume) {

            if (endSequenceDetected(scanner))
                return fToken;

        } else {
            int c= scanner.read();
            if (c == fStartSequence[0]) {
                scanner.unread();
                scanner.unread();
                int before = scanner.read();

                if (before=='\\'){
                    return Token.UNDEFINED;
                }
                scanner.read();

                if (sequenceDetected(scanner, fStartSequence, false)) {
                    if (endSequenceDetected(scanner))
                        return fToken;
                }
            }
        }

        scanner.unread();
        return Token.UNDEFINED;
    }

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    char start = (char) scanner.read();
    if ('-' !=start) {
        scanner.unread();
        return Token.UNDEFINED;
    }

    /* check if the former character is a space - if not this is not an argument */
    scanner.unread();
    scanner.unread();
    char beforeStart = (char) scanner.read();
    if (' ' !=beforeStart){
        /* no unread necessary, scanner is at start start*/
        return Token.UNDEFINED;
    }
    /* go after start again*/
    scanner.read();

    /* okay could be a parameter*/
    do {
        char c = (char) scanner.read();
        if (! isPartOfParameter(c)) {
            scanner.unread();
            break;
        }
    } while (true);
    return getSuccessToken();
}

@Test
public void line_starting_with_space_followed_by_delimiter_returns_not_token_but_undefined() {
    /* prepare */
    scanner = new SimpleTestCharacterScanner(" << EOF\nsomething\nEOF");
    rule.trace=true;

    /* execute */
    IToken result = rule.evaluate(scanner);

    /* test */
    assertEquals(Token.UNDEFINED,result);

}

private WordRule sqlSyntaxRules() {

        // Define a word rule and add SQL keywords to it.
        WordRule wordRule = new WordRule(new WordDetector(), Token.WHITESPACE, true);
        for (String reservedWord : sqlSyntax.getReservedwords()) {
            wordRule.addWord(reservedWord, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.RESERVED_WORDS)));
        }
        // TODO render unreserved keywords in the same way with reserved
        // keywords, should let user decide via preference
        for (String unreservedWord : sqlSyntax.getUnreservedwords()) {
            wordRule.addWord(unreservedWord, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.UN_RESERVED_WORDS)));
        }

        // Add the SQL datatype names to the word rule.
        for (String datatype : sqlSyntax.getTypes()) {
            wordRule.addWord(datatype, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.TYPES)));
        }

        // Add the SQL function names to the word rule.
        for (String function : sqlSyntax.getFunctions()) {
            wordRule.addWord(function, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.FUNCTIONS)));
        }

        // Add the SQL constants to the word rule.
        for (String constant : sqlSyntax.getConstants()) {
            wordRule.addWord(constant, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.CONSTANTS)));
        }

        return wordRule;
    }

IDocumentPartitioner createRecipePartitioner() {
    IPredicateRule[] rules = {
            new SingleLineRule("--", null, new Token(SQL_SINGLE_COMMENT), (char) 0, true, false), //$NON-NLS-1$
            new MultiLineRule("/*", "*/", new Token(SQL_MULTI_COMMENT), (char) 0, true),  //$NON-NLS-1$ //$NON-NLS-2$
            new MultiLineRule( "'", "'", new Token(SQL_CHARACTER_STRING_LITERAL), (char) 0 , true) //$NON-NLS-1$ //$NON-NLS-2$
    };

    RuleBasedPartitionScanner scanner = new RuleBasedPartitionScanner();
    scanner.setPredicateRules(rules);

    return new FastPartitioner(scanner, CONTENT_TYPES);
}