Java 类org.antlr.v4.runtime.BufferedTokenStream 实例源码

private List<SyntaxElement> addNonSyntaxTokens(List<SyntaxElement> syntaxElements, BufferedTokenStream tokens) {

        StackLight<SyntaxElement> regularElements = new StackLight<SyntaxElement>(syntaxElements);

        List<SyntaxElement> result = new LinkedList<SyntaxElement>();

        for (Token t : tokens.getTokens()) {
            if (!regularElements.empty() && regularElements.peek().tokenIndex == t.getTokenIndex())
                result.add(regularElements.pop());
            else {
                SyntaxElementType type = getIrregularType(t);
                if (type != SyntaxElementType.unknown)
                    result.add(SyntaxElement.create(t, type));
            }
        }

        return result;
    }

public Archetype parse(Reader reader) throws IOException {
    try {
        CharStream charStream = new ANTLRInputStream(reader);
        Lexer lexer = new adlLexer(charStream);
        adlParser parser = new adlParser(new BufferedTokenStream(lexer));
        AccumulatingErrorListener errorHandler = new AccumulatingErrorListener();
        parser.removeErrorListeners();
        parser.addErrorListener(errorHandler);
        adlParser.AdlContext context = parser.adl();
        if (!errorHandler.getErrors().isEmpty()) {
            throw new AdlParserException(Joiner.on("\n").join(errorHandler.getErrors()));
        }
        AdlTreeParser treeParser = new AdlTreeParser();
        return treeParser.parseAdl(context);

    } finally {
        reader.close();
    }
}

private void parseNodeContent(String nodeContent) throws IOException {
    YarnLexer yarnLexer = new YarnLexer(new ANTLRInputStream(nodeContent));
    YarnParser yarnParser = new YarnParser(new BufferedTokenStream(yarnLexer));
    NodeContext nodeContext = yarnParser.node();

    ParseTreeWalker parseTreeWalker = new ParseTreeWalker();
    parseTreeWalker.walk(this, nodeContext);
}

private void runTest(String filename) throws Exception {
    System.out.println("------------- " + filename + " -------------");
    YarnLexer yarnLexer = new YarnLexer(new ANTLRInputStream(YarnParserTest.class.getResourceAsStream("/" + filename)));
    YarnParser yarnParser = new YarnParser(new BufferedTokenStream(yarnLexer));
    yarnParser.setTrace(true);

    NodeContext nodeContext = yarnParser.node();

    ParseTreeWalker parseTreeWalker = new ParseTreeWalker();
    parseTreeWalker.walk(this, nodeContext);
}

public Configuration configuration() throws IOException, ConfigurationException {
    final DescriptiveErrorListener error_listener = new DescriptiveErrorListener();
    final ConfigTokenizer lexer = new ConfigTokenizer(CharStreams.fromReader(reader_));
    lexer.removeErrorListeners();
    lexer.addErrorListener(error_listener);
    final ConfigParser parser = new ConfigParser(new BufferedTokenStream(lexer));
    parser.removeErrorListeners();
    parser.addErrorListener(error_listener);
    dir_.ifPresent(parser::setDir);

    final ConfigParser.ExprContext expr;
    try {
        expr = parser.expr();
    } catch (Exception ex) {
        LOG.log(Level.SEVERE, "parser yielded exceptional return", ex);
        if (!error_listener.errors.isEmpty())
            throw new ConfigurationException(error_listener.errors, ex);
        else
            throw ex;
    }

    if (!error_listener.errors.isEmpty()) {
        if (expr.exception != null)
            throw new ConfigurationException(error_listener.errors, expr.exception);
        throw new ConfigurationException(error_listener.errors);
    } else if (expr.exception != null) {
        throw new ConfigurationException(expr.exception);
    }
    return expr.s;
}

public NameToIDSubstitution(FusionTablesSqlParser parser, BufferedTokenStream tokens,
        TableNameToIdMapper namesToIds, Map<String, String> tableAliasToName) {
    Check.notNull(tokens);
    Check.notNull(namesToIds);
    Check.notNull(tableAliasToName);

    this.mapper = namesToIds;
    rewriter = new TokenStreamRewriter(tokens);
}

public CursorContextListener(int cursorIndex, FusionTablesSqlParser parser, BufferedTokenStream tokens) {
    super(parser, tokens);
    this.cursorIndex = cursorIndex;
    this.parser = parser;

    printDebug(String.format("\ngetting context at position %d", cursorIndex));
}

ParserObjcListener(ClassDescription classDescription,
                   BufferedTokenStream tokens, ParseOptions options) {
    super();
    this.classDescription = classDescription;
    this.options = options;
    this.tokens = tokens;
    this.classDeclarations = classDescription.getmFiles();
}

@Override
public void parse(String javaSourcePath) throws SAXException, IOException {
    // parse the Java file(s) with the ANTLR generated parser
    JavaParser.CompilationUnitContext ctx = readSourceFile(javaSourcePath);
    // visit the AST with the XmlEmittingVisitor, which generates SAX events
    startDocument();
    XmlEmittingVisitor visitor = new XmlEmittingVisitor(this, (BufferedTokenStream) parser.getTokenStream());
    visitor.visit(ctx);
    endDocument();
}

/**
 * Read path matcher from reader.
 *
 * @param reader A reader supplying the input of a path expression.
 * @return A PathMatcher corresponding to the parsed input
 * from the reader.
 * @throws IOException on IO errors from the reader.
 * @throws
 * com.groupon.lex.metrics.PathMatcher.ParseException
 * on invalid path expression.
 */
public static PathMatcher valueOf(Reader reader) throws IOException, ParseException {
    class DescriptiveErrorListener extends BaseErrorListener {
        public List<String> errors = new ArrayList<>();

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                                int line, int charPositionInLine,
                                String msg, org.antlr.v4.runtime.RecognitionException e) {
            LOG.log(Level.INFO, "Parse error: {0}:{1} -> {2}", new Object[]{line, charPositionInLine, msg});
            errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg));
        }
    }

    final DescriptiveErrorListener error_listener = new DescriptiveErrorListener();

    final PathMatcherLexer lexer = new PathMatcherLexer(CharStreams.fromReader(reader));
    lexer.removeErrorListeners();
    lexer.addErrorListener(error_listener);

    final PathMatcherGrammar parser = new PathMatcherGrammar(new BufferedTokenStream(lexer));
    parser.removeErrorListeners();
    parser.addErrorListener(error_listener);

    final PathMatcherGrammar.ExprContext expr;
    try {
        expr = parser.expr();
    } catch (Exception ex) {
        LOG.log(Level.SEVERE, "parser yielded exceptional return", ex);
        if (!error_listener.errors.isEmpty())
            throw new ParseException(error_listener.errors, ex);
        else
            throw ex;
    }

    if (!error_listener.errors.isEmpty()) {
        if (expr.exception != null)
            throw new ParseException(error_listener.errors, expr.exception);
        throw new ParseException(error_listener.errors);
    } else if (expr.exception != null) {
        throw new ParseException(expr.exception);
    }
    return expr.s;
}

/**
 * Read expression from reader.
 *
 * @param reader A reader supplying the input of an expression.
 * @return A TimeSeriesMetricExpression corresponding to the parsed input
 * from the reader.
 * @throws IOException on IO errors from the reader.
 * @throws
 * com.groupon.lex.metrics.timeseries.TimeSeriesMetricExpression.ParseException
 * on invalid expression.
 */
public static TimeSeriesMetricExpression valueOf(Reader reader) throws IOException, ParseException {
    final Logger LOG = Logger.getLogger(TimeSeriesMetricExpression.class.getName());

    class DescriptiveErrorListener extends BaseErrorListener {
        public List<String> errors = new ArrayList<>();

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                                int line, int charPositionInLine,
                                String msg, org.antlr.v4.runtime.RecognitionException e) {
            LOG.log(Level.INFO, "Parse error: {0}:{1} -> {2}", new Object[]{line, charPositionInLine, msg});
            errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg));
        }
    }

    final DescriptiveErrorListener error_listener = new DescriptiveErrorListener();

    final ExpressionLexer lexer = new ExpressionLexer(CharStreams.fromReader(reader));
    lexer.removeErrorListeners();
    lexer.addErrorListener(error_listener);

    final Expression parser = new Expression(new BufferedTokenStream(lexer));
    parser.removeErrorListeners();
    parser.addErrorListener(error_listener);

    final Expression.ExprContext expr;
    try {
        expr = parser.expr();
    } catch (Exception ex) {
        LOG.log(Level.SEVERE, "parser yielded exceptional return", ex);
        if (!error_listener.errors.isEmpty())
            throw new ParseException(error_listener.errors, ex);
        else
            throw ex;
    }

    if (!error_listener.errors.isEmpty()) {
        if (expr.exception != null)
            throw new ParseException(error_listener.errors, expr.exception);
        throw new ParseException(error_listener.errors);
    } else if (expr.exception != null) {
        throw new ParseException(expr.exception);
    }
    return expr.s;
}

public SyntaxElementListener(FusionTablesSqlParser parser, BufferedTokenStream tokens)
{
    super(tokens);
    this.parser = parser;
}

public BaseFtListener(BufferedTokenStream tokens) {
    this.tokens = tokens;
}

protected ServiceParseListener(BufferedTokenStream tokens, ProtoContext context) {
    super(tokens, context);
}

ProtoParseListener(BufferedTokenStream tokens, ProtoContext context) {
    super(tokens, context);
    this.tokens = tokens;
}

protected OptionParseListener(BufferedTokenStream tokens, ProtoContext context) {
    super(tokens, context);
}

public MessageParseListener(BufferedTokenStream tokens, ProtoContext context) {
    super(tokens, context);
}

protected EnumParseListener(BufferedTokenStream tokens, ProtoContext context) {
    super(tokens, context);
}

protected AbstractProtoParserListener(BufferedTokenStream tokens, ProtoContext context) {
    this.context = context;
    this.tokens = tokens;
    usedComments = new BitSet();
}

public BlankLineListener(Printer printer, BufferedTokenStream tokenStream) {
    this.printer = printer;
    this.tokenStream = tokenStream;
}

public BraceStyleListener(Printer printer, BufferedTokenStream tokenStream) {
    this.printer = printer;
    this.tokenStream = tokenStream;
}

public PrinterListener(BufferedTokenStream tokens) {
    this.tokens = tokens;
}

public PrettyListener(BufferedTokenStream tokens) {
    this.tokens = tokens;
}

public ParserToModelExtension(final BufferedTokenStream tokens) {
  this.tokens = tokens;
}

public static HDLPackage toHDL(final BufferedTokenStream tokens, final PSHDLLang.PsModelContext ctx, final String libURI, final String src) {
  return new ParserToModelExtension(tokens).toHDLPkg(ctx, libURI, src);
}

public static HDLExpression toHDLExpression(final BufferedTokenStream tokens, final PSHDLLang.PsExpressionContext ctx) {
  IHDLObject _hDL = new ParserToModelExtension(tokens).toHDL(ctx, false);
  return ((HDLExpression) _hDL);
}

private static adlParser createAdlParser(String text) throws IOException {
    char[] chars = text.toCharArray();
    CharStream charStream = new ANTLRInputStream(chars, chars.length);
    Lexer lexer = new adlLexer(charStream);
    return new adlParser(new BufferedTokenStream(lexer));
}

XmlEmittingVisitor(ContentHandler contentHandler, BufferedTokenStream bufferedTokenStream) {
    this.contentHandler = contentHandler;
    this.bufferedTokenStream = bufferedTokenStream;
    cursor = -1;
}

/**
 * Method to get the missed HTML tag to generate more informative error message for the user.
 * This method doesn't concern itself with
 * <a href="https://www.w3.org/TR/html51/syntax.html#void-elements">void elements</a>
 * since it is forbidden to close them.
 * Missed HTML tags for the following tags will <i>not</i> generate an error message from ANTLR:
 * {@code
 * <p>
 * <li>
 * <tr>
 * <td>
 * <th>
 * <body>
 * <colgroup>
 * <dd>
 * <dt>
 * <head>
 * <html>
 * <option>
 * <tbody>
 * <thead>
 * <tfoot>
 * }
 * @param exception {@code NoViableAltException} object catched while parsing javadoc
 * @return returns appropriate {@link Token} if a HTML close tag is missed;
 *     null otherwise
 */
private static Token getMissedHtmlTag(RecognitionException exception) {
    Token htmlTagNameStart = null;
    final Interval sourceInterval = exception.getCtx().getSourceInterval();
    final List<Token> tokenList = ((BufferedTokenStream) exception.getInputStream())
            .getTokens(sourceInterval.a, sourceInterval.b);
    final Deque<Token> stack = new ArrayDeque<Token>();
    for (int i = 0; i < tokenList.size(); i++) {
        final Token token = tokenList.get(i);
        if (token.getType() == JavadocTokenTypes.HTML_TAG_NAME
                && tokenList.get(i - 1).getType() == JavadocTokenTypes.START) {
            stack.push(token);
        }
        else if (token.getType() == JavadocTokenTypes.HTML_TAG_NAME && !stack.isEmpty()) {
            if (stack.peek().getText().equals(token.getText())) {
                stack.pop();
            }
            else {
                htmlTagNameStart = stack.pop();
            }
        }
    }
    if (htmlTagNameStart == null) {
        htmlTagNameStart = stack.pop();
    }
    return htmlTagNameStart;
}

/**
 * Method to get the missed HTML tag to generate more informative error message for the user.
 * This method doesn't concern itself with
 * <a href="https://www.w3.org/TR/html51/syntax.html#void-elements">void elements</a>
 * since it is forbidden to close them.
 * Missed HTML tags for the following tags will <i>not</i> generate an error message from ANTLR:
 * {@code
 * <p>
 * <li>
 * <tr>
 * <td>
 * <th>
 * <body>
 * <colgroup>
 * <dd>
 * <dt>
 * <head>
 * <html>
 * <option>
 * <tbody>
 * <thead>
 * <tfoot>
 * }
 * @param exception {@code NoViableAltException} object catched while parsing javadoc
 * @return returns appropriate {@link Token} if a HTML close tag is missed;
 *     null otherwise
 */
private static Token getMissedHtmlTag(RecognitionException exception) {
    Token htmlTagNameStart = null;
    final Interval sourceInterval = exception.getCtx().getSourceInterval();
    final List<Token> tokenList = ((BufferedTokenStream) exception.getInputStream())
            .getTokens(sourceInterval.a, sourceInterval.b);
    final Deque<Token> stack = new ArrayDeque<>();
    for (int i = 0; i < tokenList.size(); i++) {
        final Token token = tokenList.get(i);
        if (token.getType() == JavadocTokenTypes.HTML_TAG_NAME
                && tokenList.get(i - 1).getType() == JavadocTokenTypes.START) {
            stack.push(token);
        }
        else if (token.getType() == JavadocTokenTypes.HTML_TAG_NAME && !stack.isEmpty()) {
            if (stack.peek().getText().equals(token.getText())) {
                stack.pop();
            }
            else {
                htmlTagNameStart = stack.pop();
            }
        }
    }
    if (htmlTagNameStart == null) {
        htmlTagNameStart = stack.pop();
    }
    return htmlTagNameStart;
}