@Override public TransformExpressionTree compileToExpressionTree(String expression) { CharStream charStream = new ANTLRInputStream(expression); PQL2Lexer lexer = new PQL2Lexer(charStream); lexer.setTokenFactory(new CommonTokenFactory(true)); TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer); PQL2Parser parser = new PQL2Parser(tokenStream); parser.setErrorHandler(new BailErrorStrategy()); // Parse ParseTree parseTree = parser.expression(); ParseTreeWalker walker = new ParseTreeWalker(); Pql2AstListener listener = new Pql2AstListener(expression, _splitInClause); walker.walk(listener, parseTree); final AstNode rootNode = listener.getRootNode(); return TransformExpressionTree.buildTree(rootNode); }
/** * Parses while matching an AQL expression. * * @param expression * the expression to parse * @return the corresponding {@link AstResult} */ private AstResult parseWhileAqlExpression(String expression) { final IQueryBuilderEngine.AstResult result; if (expression != null && expression.length() > 0) { AstBuilderListener astBuilder = AQL56Compatibility.createAstBuilderListener(queryEnvironment); CharStream input = new UnbufferedCharStream(new StringReader(expression), expression.length()); QueryLexer lexer = new QueryLexer(input); lexer.setTokenFactory(new CommonTokenFactory(true)); lexer.removeErrorListeners(); lexer.addErrorListener(astBuilder.getErrorListener()); TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer); QueryParser parser = new QueryParser(tokens); parser.addParseListener(astBuilder); parser.removeErrorListeners(); parser.addErrorListener(astBuilder.getErrorListener()); // parser.setTrace(true); parser.expression(); result = astBuilder.getAstResult(); } else { ErrorExpression errorExpression = (ErrorExpression) EcoreUtil .create(AstPackage.eINSTANCE.getErrorExpression()); List<org.eclipse.acceleo.query.ast.Error> errors = new ArrayList<org.eclipse.acceleo.query.ast.Error>(1); errors.add(errorExpression); final Map<Object, Integer> positions = new HashMap<Object, Integer>(); if (expression != null) { positions.put(errorExpression, Integer.valueOf(0)); } final BasicDiagnostic diagnostic = new BasicDiagnostic(); diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0, "null or empty string.", new Object[] {errorExpression })); result = new AstResult(errorExpression, positions, positions, errors, diagnostic); } return result; }
/** * Parses while matching an AQL expression. * * @param queryEnvironment * the {@link IReadOnlyQueryEnvironment} * @param type * the type to parse * @return the corresponding {@link AstResult} */ private AstResult parseWhileAqlTypeLiteral(IReadOnlyQueryEnvironment queryEnvironment, String type) { final IQueryBuilderEngine.AstResult result; if (type != null && type.length() > 0) { AstBuilderListener astBuilder = AQL56Compatibility .createAstBuilderListener((IQueryEnvironment) queryEnvironment); CharStream input = new UnbufferedCharStream(new StringReader(type), type.length()); QueryLexer lexer = new QueryLexer(input); lexer.setTokenFactory(new CommonTokenFactory(true)); lexer.removeErrorListeners(); lexer.addErrorListener(astBuilder.getErrorListener()); TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer); QueryParser parser = new QueryParser(tokens); parser.addParseListener(astBuilder); parser.removeErrorListeners(); parser.addErrorListener(astBuilder.getErrorListener()); // parser.setTrace(true); parser.typeLiteral(); result = astBuilder.getAstResult(); } else { ErrorTypeLiteral errorTypeLiteral = (ErrorTypeLiteral) EcoreUtil .create(AstPackage.eINSTANCE.getErrorTypeLiteral()); List<org.eclipse.acceleo.query.ast.Error> errors = new ArrayList<org.eclipse.acceleo.query.ast.Error>(1); errors.add(errorTypeLiteral); final Map<Object, Integer> positions = new HashMap<Object, Integer>(); if (type != null) { positions.put(errorTypeLiteral, Integer.valueOf(0)); } final BasicDiagnostic diagnostic = new BasicDiagnostic(); diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0, "null or empty type.", new Object[] {errorTypeLiteral })); result = new AstResult(errorTypeLiteral, positions, positions, errors, diagnostic); } return result; }
public static StringTemplate fromString(String pattern) { class DescriptiveErrorListener extends BaseErrorListener { public List<String> errors = new ArrayList<>(); @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg)); } } final DescriptiveErrorListener error_listener = new DescriptiveErrorListener(); final StringSubstitutionLexer lexer = new StringSubstitutionLexer(CharStreams.fromString(pattern)); lexer.removeErrorListeners(); lexer.addErrorListener(error_listener); final StringSubstitutionParser parser = new StringSubstitutionParser(new UnbufferedTokenStream(lexer)); parser.removeErrorListeners(); parser.addErrorListener(error_listener); parser.setErrorHandler(new BailErrorStrategy()); final StringSubstitutionParser.ExprContext result = parser.expr(); if (result.exception != null) throw new IllegalArgumentException("errors during parsing: " + pattern, result.exception); else if (!error_listener.errors.isEmpty()) throw new IllegalArgumentException("syntax errors during parsing:\n" + String.join("\n", error_listener.errors.stream().map(s -> " " + s).collect(Collectors.toList()))); return result.s; }
public static Map<String, Any2<String, Number>> parse(String pattern) { class DescriptiveErrorListener extends BaseErrorListener { public List<String> errors = new ArrayList<>(); @Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg)); } } final DescriptiveErrorListener error_listener = new DescriptiveErrorListener(); final CollectdTagsLexer lexer = new CollectdTagsLexer(new ANTLRInputStream(pattern)); lexer.removeErrorListeners(); lexer.addErrorListener(error_listener); final CollectdTagsParser parser = new CollectdTagsParser(new UnbufferedTokenStream(lexer)); parser.removeErrorListeners(); parser.addErrorListener(error_listener); parser.setErrorHandler(new BailErrorStrategy()); final CollectdTagsParser.ExprContext result = parser.expr(); if (result.exception != null) throw new IllegalArgumentException("errors during parsing: " + pattern, result.exception); else if (!error_listener.errors.isEmpty()) throw new IllegalArgumentException("syntax errors during parsing:\n" + String.join("\n", error_listener.errors.stream().map(s -> " " + s).collect(Collectors.toList()))); return result.result; }
private static ParseTree parseSQL(String sql) { CharStream charStream = new ANTLRInputStream(sql); RQLLexer lexer = new RQLLexer(charStream); lexer.setTokenFactory(new CommonTokenFactory(true)); TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer); RQLParser parser = new RQLParser(tokenStream); parser.setErrorHandler(new BailErrorStrategy()); return parser.root(); }
/** * @param reader a reader representing several queries * @return a list of queries */ @Override public <T extends Query<?>> Stream<T> parseList(Reader reader) { UnbufferedCharStream charStream = new UnbufferedCharStream(reader); GraqlErrorListener errorListener = GraqlErrorListener.withoutQueryString(); GraqlLexer lexer = createLexer(charStream, errorListener); /* We tell the lexer to copy the text into each generated token. Normally when calling `Token#getText`, it will look into the underlying `TokenStream` and call `TokenStream#size` to check it is in-bounds. However, `UnbufferedTokenStream#size` is not supported (because then it would have to read the entire input). To avoid this issue, we set this flag which will copy over the text into each `Token`, s.t. that `Token#getText` will just look up the copied text field. */ lexer.setTokenFactory(new CommonTokenFactory(true)); // Use an unbuffered token stream so we can handle extremely large input strings UnbufferedTokenStream tokenStream = new UnbufferedTokenStream(ChannelTokenSource.of(lexer)); GraqlParser parser = createParser(tokenStream, errorListener); /* The "bail" error strategy prevents us reading all the way to the end of the input, e.g.
match $x isa person; insert $x has name "Bob"; match $x isa movie; get; ^ ``` In this example, when ANTLR reaches the indicated `match`, it considers two possibilities: 1. this is the end of the query 2. the user has made a mistake. Maybe they accidentally pasted the `match` here. Because of case 2, ANTLR will parse beyond the `match` in order to produce a more helpful error message. This causes memory issues for very large queries, so we use the simpler "bail" strategy that will immediately stop when it hits `match`. */ parser.setErrorHandler(new BailErrorStrategy()); // This is a lazy iterator that will only consume a single query at a time, without parsing any further. // This means it can pass arbitrarily long streams of queries in constant memory! Iterable<T> queryIterator = () -> new AbstractIterator<T>() { @Nullable @Override protected T computeNext() { int latestToken = tokenStream.LA(1); if (latestToken == Token.EOF) { endOfData(); return null; } else { // This will parse and consume a single query, even if it doesn't reach an EOF // When we next run it, it will start where it left off in the stream return (T) QUERY.parse(parser, errorListener); } } }; return StreamSupport.stream(queryIterator.spliterator(), false);
}
```
