@Override public IToken nextToken() { if (currentTemplateTextToken != null) { if (currentTemplateTextToken.hasNext()) return currentTemplateTextToken.nextToken(); else currentTemplateTextToken = null; } if (!getIterator().hasNext()) return Token.EOF; ILexerTokenRegion next = getIterator().next(); int tokenType = next.getLexerTokenType(); switch (tokenType) { case RULE_TEMPLATE_HEAD: case RULE_TEMPLATE_MIDDLE: { currentTemplateTextToken = createTemplateToken(tokenType, next); return currentTemplateTextToken.nextToken(); } default: setCurrentToken(next); return createToken(next); } }
@Override public IToken evaluate(ICharacterScanner scanner1, boolean resume) { if (!(scanner1 instanceof BatchDocumentPartitionScanner)){ return Token.UNDEFINED; } BatchDocumentPartitionScanner scanner = (BatchDocumentPartitionScanner)scanner1; if (scanner.getOffset()<ECHO.length()){ /* would be a problem on start on going back...*/ return Token.UNDEFINED; } scanner.unread(); // go to character before int charBeforeAsInt = scanner.read(); // get character, moves to origin pos again /* fast guard closing:*/ if (charBeforeAsInt!='o' && charBeforeAsInt!='O'){ // echo ECHO <- last character must be an o other wise guard close... return Token.UNDEFINED; } ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner); if (parserSupport.isAfterEchoHandled(codePosSupport)){ return getSuccessToken(); } codePosSupport.resetToStartPos(); return Token.UNDEFINED; }
@Override public IToken evaluate(ICharacterScanner scanner, boolean resume) { int r = scanner.read(); scanner.unread(); /* fast guard closing:*/ if (ICharacterScanner.EOF==r || r!='<'){ return Token.UNDEFINED; } ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner); if (hereStringSupport.isHereStringStateHandled(codePosSupport)){ return getSuccessToken(); } codePosSupport.resetToStartPos(); return Token.UNDEFINED; }
@Override public IToken evaluate(ICharacterScanner scanner, boolean resume) { int r = scanner.read(); scanner.unread(); /* fast guard closing:*/ if (ICharacterScanner.EOF==r || r!='<'){ return Token.UNDEFINED; } ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner); if (hereDocSupport.isHereDocStateHandled(codePosSupport)){ return getSuccessToken(); } codePosSupport.resetToStartPos(); return Token.UNDEFINED; }
public RelationModelPartitionScanner() { final IToken commentPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_COMMENT); final IToken optionPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_OPTION); final IToken universePartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_UNIVERSE); final IToken relBoundPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_REL_BOUND); final List<IRule> rules = new ArrayList<IRule>(); rules.add(new MultiLineRule("/**", "**/", commentPartition)); rules.add(new EndOfLineRule("--", commentPartition)); // rules.add(new EndOfLineRule("//", commentPartition)); rules.add(new MultiLineRule("options {", "}", optionPartition)); rules.add(new MultiLineRule("universe {", "}", universePartition)); rules.add(new MultiLineRule("universe [", "]", universePartition)); rules.add(new RelationBoundsRule("relations", "}", relBoundPartition)); final IPredicateRule[] result = new IPredicateRule[rules.size()]; rules.toArray(result); this.setPredicateRules(result); }
@Override protected IToken doEvaluate(final ICharacterScanner scanner, final boolean resume) { if (resume) { if (this.endSequenceDetected(scanner)) { return this.fToken; } } else { final int c = scanner.read(); if (c == this.fStartSequence[0]) { if (this.sequenceDetected(scanner, this.fStartSequence, false)) { if (this.endSequenceDetected(scanner)) { return this.fToken; } } } } scanner.unread(); return Token.UNDEFINED; }
public ModelScanner(final ColorManager colorManager) { final IToken bracketsToken = new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.BRACKETS))); final IToken commToken = new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.COMMENT))); final List<IRule> rules = new ArrayList<>(); rules.add(new WhitespaceRule(new ValidationWhitespaceDetector())); rules.add(new BracketsRule(bracketsToken)); rules.add(new MultiLineRule("/**", "**/", commToken)); rules.add(new EndOfLineRule("--", commToken)); final IRule[] result = new IRule[rules.size()]; rules.toArray(result); this.setRules(result); }
public EiffelPartitionScanner() { super(); IToken multilineComment = new Token(MULTILINE_COMMENT); IToken singlelineComment = new Token(SINGLELINE_COMMENT); IToken string = new Token(STRING); List<Object> rules = new ArrayList<>(); //Add rule for single line comments. rules.add(new EndOfLineRule("--", singlelineComment)); // Add rule for strings and character constants. rules.add(new SingleLineRule("\"", "\"", string, '\\')); rules.add(new SingleLineRule("'", "'", string, '\\')); //Add rules for multi-line comments and eiffeldocs rules.add(new MultiLineRule("--[[", "]]", multilineComment, (char) 0, true)); IPredicateRule[] result = new IPredicateRule[rules.size()]; rules.toArray(result); setPredicateRules(result); }
public XMLTextScanner( ColorManager colorManager) { ESCAPED_CHAR = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.ESCAPED_CHAR))); CDATA_START = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA))); CDATA_END = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA))); CDATA_TEXT = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA_TEXT))); IRule[] rules = new IRule[2]; // Add rule to pick up escaped chars // Add rule to pick up start of CDATA section rules[0] = new CDataRule(CDATA_START, true); // Add a rule to pick up end of CDATA sections rules[1] = new CDataRule(CDATA_END, false); setRules(rules); }
public XMLPartitionScanner() { IToken xmlComment = new Token(XML_COMMENT); IToken xmlPI = new Token(XML_PI); IToken startTag = new Token(XML_START_TAG); IToken endTag = new Token(XML_END_TAG); IToken docType = new Token(XML_DOCTYPE); IToken text = new Token(XML_TEXT); IPredicateRule[] rules = new IPredicateRule[7]; rules[0] = new NonMatchingRule(); rules[1] = new MultiLineRule("<!--", "-->", xmlComment); rules[2] = new MultiLineRule("<?", "?>", xmlPI); rules[3] = new MultiLineRule("</", ">", endTag); rules[4] = new StartTagRule(startTag); rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType); rules[6] = new XMLTextPredicateRule(text); setPredicateRules(rules); }
public IToken evaluate( ICharacterScanner scanner){ buffer.setLength(0); int c = read(scanner); if (c == '&') { int i = 0; do { c = read(scanner); i++; if (c == '<' || c == ']') { for (int j = i - 1; j > 0; j--) scanner.unread(); return Token.UNDEFINED; } } while (c != ';'); return fToken; } scanner.unread(); return Token.UNDEFINED; }
public IToken evaluate( ICharacterScanner scanner){ buffer.setLength(0); charsRead = 0; int c = read(scanner); if (c == matchString.charAt(0)) { do { c = read(scanner); } while (isOK((char) c)); if (charsRead == matchString.length()) { return fToken; } else { rewind(scanner); return Token.UNDEFINED; } } scanner.unread(); return Token.UNDEFINED; }
private SqlCommentPartitionScanner() { IToken commentToken = new Token(SQL_COMMENT); List<IPredicateRule> rules= new ArrayList<>(); rules.add(new EndOfLineRule("--", commentToken, '\\')); rules.add(new MultiLineRule("/*", "*/", commentToken, '\\')); rules.add(new WordPredicateRule(commentToken)); rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\')); rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\')); IPredicateRule[] result= new IPredicateRule[rules.size()]; rules.toArray(result); setPredicateRules(result); }
/** * Does the actual evaluation of the stream. * * @param scanner The scanner * @param count The initial count of { * @return <code>fToken</code> on success, <code>Token.UNDEFINED</code> if * the match doesn't succeed */ private IToken doEvaluate(ICharacterScanner scanner, int count) { boolean inString = false; int c = scanner.read(); if (((char) c) == '{') { do { c = scanner.read(); if (((char) c) == '{' && !inString) count++; else if (((char) c) == '}' && !inString) count--; else if (((char) c) == '"' && !inString) inString = true; else if (((char) c) == '"' && inString) inString = false; else if (c == ICharacterScanner.EOF) return Token.UNDEFINED; } while (count > 0); return fToken; } scanner.unread(); return Token.UNDEFINED; }
public IToken evaluate(ICharacterScanner scanner, boolean resume) { if (resume) { boolean inString = false; do { int c = scanner.read(); if (((char) c) == ',' && !inString) break; else if (((char) c) == '@') { scanner.unread(); return Token.UNDEFINED; } else if (((char) c) == '"' && !inString) inString = true; else if (((char) c) == '"' && inString) inString = false; else if (c == ICharacterScanner.EOF) return Token.UNDEFINED; } while (true); } return doEvaluate(scanner, 1); }
public IToken evaluate(ICharacterScanner scanner) { int count = 0; int charsRead = 0; int c = scanner.read(); if (((char) c) == '"') { do { c = scanner.read(); charsRead++; if (c == ICharacterScanner.EOF) { unwind(scanner, charsRead); return Token.UNDEFINED; } else if (((char) c) == '{') { count++; } else if (((char) c) == '}') { if (count == 0) { unwind(scanner, charsRead); return Token.UNDEFINED; } count--; } } while (((char) c) != '"'); return fToken; } scanner.unread(); return Token.UNDEFINED; }
/** * Creates the partitioner and sets up the appropriate rules. */ public BibPartitionScanner() { super(); IToken bibEntry = new Token(BIB_ENTRY); List rules = new ArrayList(); // Add rule for single line comments. // rules.add(new EndOfLineRule("//", Token.UNDEFINED)); // Add rule for strings and character constants. // rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\')); // rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\')); // Add rules for BibTeX entries //rules.add(new MultiLineRule("{", "}", bibEntry, (char) 0, false)); rules.add(new BibBraceRule(true, bibEntry));; IPredicateRule[] result= new IPredicateRule[rules.size()]; rules.toArray(result); setPredicateRules(result); }
@Override public IToken evaluate(ICharacterScanner scanner) { found = false; int c = scanner.read(); scanner.unread(); if (isHws(c)) { int col = scanner.getColumn(); if (col == 1) { scanner.unread(); int p = scanner.read(); if (p == '\n') col = 0; } if (col == 0) { tabWidth = FluentMkUI.getDefault().getPreferenceStore().getInt(Prefs.EDITOR_TAB_WIDTH); if (tabWidth < 2) tabWidth = 4; line = 0; while (evaluateLine(scanner)) { found = true; line++; } } } if (found) return token; return Token.UNDEFINED; }
/** * A default constructor. * @param manager */ public TexArgScanner(ColorManager manager) { IToken commentToken = new Token(new TextAttribute(manager .getColor(ColorManager.COMMENT), null, manager.getStyle(ColorManager.COMMENT_STYLE))); //Commands are colored in argument color with command styles IToken commandToken = new Token( new TextAttribute( manager.getColor(ColorManager.CURLY_BRACKETS), null, manager.getStyle(ColorManager.COMMAND_STYLE))); List<IRule> rules = new ArrayList<IRule>(); rules.add(new EndOfLineRule("%", commentToken, '\\')); rules.add(new WhitespaceRule(new WhitespaceDetector())); rules.add(new WordRule(new TexWord(), commandToken)); IRule[] result = new IRule[rules.size()]; rules.toArray(result); setRules(result); }
/** * A default constructor. * @param manager */ public TexOptArgScanner(ColorManager manager) { IToken commentToken = new Token(new TextAttribute(manager .getColor(ColorManager.COMMENT), null, manager.getStyle(ColorManager.COMMENT_STYLE))); //Commands are colored in argument color with command styles IToken commandToken = new Token( new TextAttribute( manager.getColor(ColorManager.SQUARE_BRACKETS), null, manager.getStyle(ColorManager.COMMAND_STYLE))); List<IRule> rules = new ArrayList<IRule>(); rules.add(new EndOfLineRule("%", commentToken, '\\')); rules.add(new WhitespaceRule(new WhitespaceDetector())); rules.add(new WordRule(new TexWord(), commandToken)); IRule[] result = new IRule[rules.size()]; rules.toArray(result); setRules(result); }
@Override public IToken evaluate(ICharacterScanner scanner) { int c = scanner.read(); if (c != ICharacterScanner.EOF && 0 == scanner.getColumn() - 1) { try { if (dotStarts(scanner, c)) { gotoEOB(scanner, c); return getSuccessToken(); } } catch (BadLocationException e) { Log.error("DotCodeRule", e); } } scanner.unread(); return Token.UNDEFINED; }
public XMLTextScanner(ColorManager colorManager) { ESCAPED_CHAR = new Token(new TextAttribute(colorManager .getColor(IXMLColorConstants.ESCAPED_CHAR))); CDATA_START = new Token(new TextAttribute(colorManager .getColor(IXMLColorConstants.CDATA))); CDATA_END = new Token(new TextAttribute(colorManager .getColor(IXMLColorConstants.CDATA))); CDATA_TEXT = new Token(new TextAttribute(colorManager .getColor(IXMLColorConstants.CDATA_TEXT))); IRule[] rules = new IRule[2]; // Add rule to pick up escaped chars // Add rule to pick up start of CDATA section //rules[0] = new CDataRule(CDATA_START, true); // Add a rule to pick up end of CDATA sections //rules[1] = new CDataRule(CDATA_END, false); setRules(rules); }
public XMLPartitionScanner() { IToken xmlComment = new Token(XML_COMMENT); IToken xmlPI = new Token(XML_PI); IToken startTag = new Token(XML_START_TAG); IToken endTag = new Token(XML_END_TAG); IToken docType = new Token(XML_DOCTYPE); // IToken text = new Token(XML_TEXT); // IToken tag = new Token(XML_TAG); IPredicateRule[] rules = new IPredicateRule[6]; rules[0] = new NonMatchingRule(); rules[1] = new MultiLineRule("<!--", "-->", xmlComment, '\\', true); rules[2] = new MultiLineRule("<?", "?>", xmlPI); rules[3] = new MultiLineRule("</", ">", endTag); rules[4] = new StartTagRule(startTag); rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType); // rules[6] = new XMLTextPredicateRule(text); // rules[7] = new TagRule(tag); setPredicateRules(rules); }
@Override public IToken evaluate(ICharacterScanner scanner) { int col = scanner.getColumn(); if (col == 1) { scanner.unread(); int p = scanner.read(); if (p == '\n') col = 0; } try { if (col == 0) { String text = getScanText(scanner); if (tagStarts(text)) { gotoEOB(scanner); return getSuccessToken(); } } } catch (BadLocationException e) { Log.error("HtmlCodeRule", e); } return Token.UNDEFINED; }
public IToken evaluate(ICharacterScanner scanner) { buffer.setLength(0); int c = read(scanner); if (c == '&') { int i = 0; do { c = read(scanner); i++; if (c == '<' || c == ']') { for (int j = i - 1; j > 0; j--) scanner.unread(); return Token.UNDEFINED; } } while (c != ';'); return fToken; } scanner.unread(); return Token.UNDEFINED; }
public IToken evaluate(ICharacterScanner scanner) { buffer.setLength(0); charsRead = 0; int c = read(scanner); if (c == matchString.charAt(0)) { do { c = read(scanner); } while (isOK((char) c)); if (charsRead == matchString.length()) { return fToken; } else { rewind(scanner); return Token.UNDEFINED; } } scanner.unread(); return Token.UNDEFINED; }
@Override public IToken evaluate(ICharacterScanner scanner, boolean resume) { if (fColumn == UNDEFINED) return doEvaluate(scanner, resume); int c = scanner.read(); scanner.unread(); if (c == fStartSequence[0]) { int col = scanner.getColumn(); if (col == 1) { scanner.unread(); int p = scanner.read(); if (p == '\n') col = 0; } return (fColumn == col ? doEvaluate(scanner, resume) : Token.UNDEFINED); } return Token.UNDEFINED; }
private void adaptToColorChange(Token token, PropertyChangeEvent event) { RGB rgb = null; Object value = event.getNewValue(); if (value instanceof RGB) { rgb = (RGB) value; } else if (value instanceof String) { rgb = StringConverter.asRGB((String) value); } if (rgb != null) { String property = event.getProperty(); Color color = colorMgr.getColor(property); if (!rgb.equals(color.getRGB())) { colorMgr.setValue(property, rgb); color = colorMgr.getColor(property); } Object data = token.getData(); if (data instanceof TextAttribute) { TextAttribute oldAttr = (TextAttribute) data; token.setData(new TextAttribute(color, oldAttr.getBackground(), oldAttr.getStyle())); } } }
private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) { boolean eventValue = false; Object value = event.getNewValue(); if (value instanceof Boolean) eventValue = ((Boolean) value).booleanValue(); else if (IPreferenceStore.TRUE.equals(value)) eventValue = true; Object data = token.getData(); if (data instanceof TextAttribute) { TextAttribute oldAttr = (TextAttribute) data; boolean activeValue = (oldAttr.getStyle() & styleAttribute) == styleAttribute; if (activeValue != eventValue) token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(), eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute)); } }
protected IToken createToken() { String attributeId = delimiter ? HighlightingConfiguration.TEMPLATE_DELIMITER_ID : HighlightingConfiguration.TEMPLATE_ID; Token token = new Token(getAttribute(attributeId)); return token; }
protected IToken doEvaluate(ICharacterScanner scanner, boolean resume) { if (resume) { if (endSequenceDetected(scanner)) return fToken; } else { int c= scanner.read(); if (c == fStartSequence[0]) { scanner.unread(); scanner.unread(); int before = scanner.read(); if (before=='\\'){ return Token.UNDEFINED; } scanner.read(); if (sequenceDetected(scanner, fStartSequence, false)) { if (endSequenceDetected(scanner)) return fToken; } } } scanner.unread(); return Token.UNDEFINED; }
@Override public IToken evaluate(ICharacterScanner scanner, boolean resume) { char start = (char) scanner.read(); if ('-' !=start) { scanner.unread(); return Token.UNDEFINED; } /* check if the former character is a space - if not this is not an argument */ scanner.unread(); scanner.unread(); char beforeStart = (char) scanner.read(); if (' ' !=beforeStart){ /* no unread necessary, scanner is at start start*/ return Token.UNDEFINED; } /* go after start again*/ scanner.read(); /* okay could be a parameter*/ do { char c = (char) scanner.read(); if (! isPartOfParameter(c)) { scanner.unread(); break; } } while (true); return getSuccessToken(); }
protected XMLScanner getXMLScanner() { if (scanner == null) { scanner = new XMLScanner(colorManager); scanner.setDefaultReturnToken(new Token(new TextAttribute( colorManager.getColor(IXMLColorConstants.DEFAULT)))); } return scanner; }
protected XMLTagScanner getXMLTagScanner() { if (tagScanner == null) { tagScanner = new XMLTagScanner(colorManager); tagScanner.setDefaultReturnToken(new Token(new TextAttribute( colorManager.getColor(IXMLColorConstants.TAG)))); } return tagScanner; }
public XMLTagScanner(ColorManager manager) { IToken string = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.STRING))); IRule[] rules = new IRule[3]; // Add rule for double quotes rules[0] = new SingleLineRule("\"", "\"", string, '\\'); // Add a rule for single quotes rules[1] = new SingleLineRule("'", "'", string, '\\'); // Add generic whitespace rule. rules[2] = new WhitespaceRule(new XMLWhitespaceDetector()); setRules(rules); }
@Test public void line_starting_with_space_followed_by_delimiter_returns_not_token_but_undefined() { /* prepare */ scanner = new SimpleTestCharacterScanner(" << EOF\nsomething\nEOF"); rule.trace=true; /* execute */ IToken result = rule.evaluate(scanner); /* test */ assertEquals(Token.UNDEFINED,result); }
private WordRule sqlSyntaxRules() { // Define a word rule and add SQL keywords to it. WordRule wordRule = new WordRule(new WordDetector(), Token.WHITESPACE, true); for (String reservedWord : sqlSyntax.getReservedwords()) { wordRule.addWord(reservedWord, new Token( getTextAttribute(prefs, SQLEditorStatementTypes.RESERVED_WORDS))); } // TODO render unreserved keywords in the same way with reserved // keywords, should let user decide via preference for (String unreservedWord : sqlSyntax.getUnreservedwords()) { wordRule.addWord(unreservedWord, new Token( getTextAttribute(prefs, SQLEditorStatementTypes.UN_RESERVED_WORDS))); } // Add the SQL datatype names to the word rule. for (String datatype : sqlSyntax.getTypes()) { wordRule.addWord(datatype, new Token( getTextAttribute(prefs, SQLEditorStatementTypes.TYPES))); } // Add the SQL function names to the word rule. for (String function : sqlSyntax.getFunctions()) { wordRule.addWord(function, new Token( getTextAttribute(prefs, SQLEditorStatementTypes.FUNCTIONS))); } // Add the SQL constants to the word rule. for (String constant : sqlSyntax.getConstants()) { wordRule.addWord(constant, new Token( getTextAttribute(prefs, SQLEditorStatementTypes.CONSTANTS))); } return wordRule; }
IDocumentPartitioner createRecipePartitioner() { IPredicateRule[] rules = { new SingleLineRule("--", null, new Token(SQL_SINGLE_COMMENT), (char) 0, true, false), //$NON-NLS-1$ new MultiLineRule("/*", "*/", new Token(SQL_MULTI_COMMENT), (char) 0, true), //$NON-NLS-1$ //$NON-NLS-2$ new MultiLineRule( "'", "'", new Token(SQL_CHARACTER_STRING_LITERAL), (char) 0 , true) //$NON-NLS-1$ //$NON-NLS-2$ }; RuleBasedPartitionScanner scanner = new RuleBasedPartitionScanner(); scanner.setPredicateRules(rules); return new FastPartitioner(scanner, CONTENT_TYPES); }