我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用parser.suite()。
def isparseable(self, deindent=True): """ return True if source is parseable, heuristically deindenting it by default. """ try: import parser except ImportError: syntax_checker = lambda x: compile(x, 'asd', 'exec') else: syntax_checker = parser.suite if deindent: source = str(self.deindent()) else: source = str(self) try: #compile(source+'\n', "x", "exec") syntax_checker(source+'\n') except KeyboardInterrupt: raise except Exception: return False else: return True
def check_pre_py_listing(self, dom, filename): for node in domhelpers.findNodesNamed(dom, 'pre'): if node.getAttribute('class') == 'python': try: text = domhelpers.getNodeText(node) # Fix < and > text = text.replace('>', '>').replace('<', '<') # Strip blank lines lines = filter(None,[l.rstrip() for l in text.split('\n')]) # Strip leading space while not [1 for line in lines if line[:1] not in ('',' ')]: lines = [line[1:] for line in lines] text = '\n'.join(lines) + '\n' try: parser.suite(text) except parserErrors, e: # Pretend the "..." idiom is syntactically valid text = text.replace("...","'...'") parser.suite(text) except parserErrors, e: self._reportError(filename, node, 'invalid python code:' + str(e))
def classdef(self, nodelist): # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite name = nodelist[1][1] doc = self.get_docstring(nodelist[-1]) if nodelist[2][0] == token.COLON: bases = [] elif nodelist[3][0] == token.RPAR: bases = [] else: bases = self.com_bases(nodelist[3]) # code for class code = self.com_node(nodelist[-1]) if doc is not None: assert isinstance(code, Stmt) assert isinstance(code.nodes[0], Discard) del code.nodes[0] return Class(name, bases, doc, code, lineno=nodelist[1][2])
def testChunk(t, fileName): global _numFailed print '----', fileName, try: st = parser.suite(t) tup = parser.st2tuple(st) # this discards the first ST; a huge memory savings when running # against a large source file like Tkinter.py. st = None new = parser.tuple2st(tup) except parser.ParserError, err: print print 'parser module raised exception on input file', fileName + ':' traceback.print_exc() _numFailed = _numFailed + 1 else: if tup != parser.st2tuple(new): print print 'parser module failed on input file', fileName _numFailed = _numFailed + 1 else: print 'o.k.'
def test_define_keywords(self): import parser grammar = ''' @@keyword :: B C @@keyword :: 'A' start = ('a' 'b').{'x'}+ ; ''' model = compile(grammar, "test") c = codegen(model) parser.suite(c) grammar2 = str(model) model2 = compile(grammar2, "test") c2 = codegen(model2) parser.suite(c2) self.assertEqual(grammar2, str(model2))
def test_check_keywords(self): import parser grammar = ''' @@keyword :: A start = {id}+ $ ; @name id = /\w+/ ; ''' model = compile(grammar, 'test') c = codegen(model) parser.suite(c) ast = model.parse('hello world') self.assertEqual(['hello', 'world'], ast) try: ast = model.parse("hello A world") self.assertEqual(['hello', 'A', 'world'], ast) self.fail('accepted keyword as name') except FailedParse as e: self.assertTrue('"A" is a reserved word' in str(e)) pass
def _loadfile(self, fileno): try: filename = self._filemap[fileno] except KeyError: print "Could not identify fileId", fileno return 1 if filename is None: return 1 absname = os.path.normcase(os.path.join(self.cwd, filename)) try: fp = open(absname) except IOError: return st = parser.suite(fp.read()) fp.close() # Scan the tree looking for def and lambda nodes, filling in # self._funcmap with all the available information. funcdef = symbol.funcdef lambdef = symbol.lambdef stack = [st.totuple(1)] while stack: tree = stack.pop() try: sym = tree[0] except (IndexError, TypeError): continue if sym == funcdef: self._funcmap[(fileno, tree[2][2])] = filename, tree[2][1] elif sym == lambdef: self._funcmap[(fileno, tree[1][2])] = filename, "<lambda>" stack.extend(list(tree[1:]))
def parsesuite(self, text): """Return a modified parse tree for the given suite text.""" return self.transform(parser.suite(text))
def if_stmt(self, nodelist): # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] tests = [] for i in range(0, len(nodelist) - 3, 4): testNode = self.com_node(nodelist[i + 1]) suiteNode = self.com_node(nodelist[i + 3]) tests.append((testNode, suiteNode)) if len(nodelist) % 4 == 3: elseNode = self.com_node(nodelist[-1]) ## elseNode.lineno = nodelist[-1][1][2] else: elseNode = None return If(tests, elseNode, lineno=nodelist[0][2])
def while_stmt(self, nodelist): # 'while' test ':' suite ['else' ':' suite] testNode = self.com_node(nodelist[1]) bodyNode = self.com_node(nodelist[3]) if len(nodelist) > 4: elseNode = self.com_node(nodelist[6]) else: elseNode = None return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
def for_stmt(self, nodelist): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] assignNode = self.com_assign(nodelist[1], OP_ASSIGN) listNode = self.com_node(nodelist[3]) bodyNode = self.com_node(nodelist[5]) if len(nodelist) > 8: elseNode = self.com_node(nodelist[8]) else: elseNode = None return For(assignNode, listNode, bodyNode, elseNode, lineno=nodelist[0][2])
def suite(self, nodelist): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT if len(nodelist) == 1: return self.com_stmt(nodelist[0]) stmts = [] for node in nodelist: if node[0] == symbol.stmt: self.com_append_stmt(stmts, node) return Stmt(stmts) # -------------------------------------------------------------- # # EXPRESSION NODES (invoked by com_node()) #
def com_with(self, nodelist): # with_stmt: 'with' with_item (',' with_item)* ':' suite body = self.com_node(nodelist[-1]) for i in range(len(nodelist) - 3, 0, -2): ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) if i == 1: return ret body = ret
def get_docstring(self, node, n=None): if n is None: n = node[0] node = node[1:] if n == symbol.suite: if len(node) == 1: return self.get_docstring(node[0]) for sub in node: if sub[0] == symbol.stmt: return self.get_docstring(sub) return None if n == symbol.file_input: for sub in node: if sub[0] == symbol.stmt: return self.get_docstring(sub) return None if n == symbol.atom: if node[0][0] == token.STRING: s = '' for t in node: s = s + eval(t[1]) return s return None if n == symbol.stmt or n == symbol.simple_stmt \ or n == symbol.small_stmt: return self.get_docstring(node[0]) if n in _doc_nodes and len(node) == 1: return self.get_docstring(node[0]) return None
def test_flags_passed(self): # The unicode literals flags has to be passed from the paser to AST # generation. suite = parser.suite("from __future__ import unicode_literals; x = ''") code = suite.compile() scope = {} exec(code, {}, scope) self.assertIsInstance(scope["x"], str)
def check_suite(self, s): self.roundtrip(parser.suite, s)
def test_position(self): # An absolutely minimal test of position information. Better # tests would be a big project. code = "def f(x):\n return x + 1" st1 = parser.suite(code) st2 = st1.totuple(line_info=1, col_info=1) def walk(tree): node_type = tree[0] next = tree[1] if isinstance(next, tuple): for elt in tree[1:]: for x in walk(elt): yield x else: yield tree terminals = list(walk(st2)) self.assertEqual([ (1, 'def', 1, 0), (1, 'f', 1, 4), (7, '(', 1, 5), (1, 'x', 1, 6), (8, ')', 1, 7), (11, ':', 1, 8), (4, '', 1, 9), (5, '', 2, -1), (1, 'return', 2, 4), (1, 'x', 2, 11), (14, '+', 2, 13), (2, '1', 2, 15), (4, '', 2, 16), (6, '', 2, -1), (4, '', 2, -1), (0, '', 2, -1)], terminals)
def test_compile_suite(self): st = parser.suite('x = 2; y = x + 3') code = parser.compilest(st) globs = {} exec(code, globs) self.assertEqual(globs['y'], 5)
def test_compile_error(self): st = parser.suite('1 = 3 + 4') self.assertRaises(SyntaxError, parser.compilest, st)
def suite(self, text): # Python 3 #f = io.StringIO(text) f = cStringIO.StringIO(text) tokens = tokenize.generate_tokens(f.readline) tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol) return tree
def test_flags_passed(self): # The unicode literals flags has to be passed from the paser to AST # generation. suite = parser.suite("from __future__ import unicode_literals; x = ''") code = suite.compile() scope = {} exec code in scope self.assertIsInstance(scope["x"], unicode)
def test_compile_suite(self): st = parser.suite('x = 2; y = x + 3') code = parser.compilest(st) globs = {} exec code in globs self.assertEqual(globs['y'], 5)
def test_compile_badunicode(self): st = parser.suite('a = u"\U12345678"') self.assertRaises(SyntaxError, parser.compilest, st) st = parser.suite('a = u"\u1"') self.assertRaises(SyntaxError, parser.compilest, st)
def test_sizeof(self): def XXXROUNDUP(n): if n <= 1: return n if n <= 128: return (n + 3) & ~3 return 1 << (n - 1).bit_length() basesize = support.calcobjsize('Pii') nodesize = struct.calcsize('hP3iP0h') def sizeofchildren(node): if node is None: return 0 res = 0 hasstr = len(node) > 1 and isinstance(node[-1], str) if hasstr: res += len(node[-1]) + 1 children = node[1:-1] if hasstr else node[1:] if children: res += XXXROUNDUP(len(children)) * nodesize for child in children: res += sizeofchildren(child) return res def check_st_sizeof(st): self.check_sizeof(st, basesize + nodesize + sizeofchildren(st.totuple())) check_st_sizeof(parser.expr('2 + 3')) check_st_sizeof(parser.expr('2 + 3 + 4')) check_st_sizeof(parser.suite('x = 2 + 3')) check_st_sizeof(parser.suite('')) check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-')) check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']')) # XXX tests for pickling and unpickling of ST objects should go here
def funcdef(self, nodelist): # -6 -5 -4 -3 -2 -1 # funcdef: [decorators] 'def' NAME parameters ':' suite # parameters: '(' [varargslist] ')' if len(nodelist) == 6: assert nodelist[0][0] == symbol.decorators decorators = self.decorators(nodelist[0][1:]) else: assert len(nodelist) == 5 decorators = None lineno = nodelist[-4][2] name = nodelist[-4][1] args = nodelist[-3][2] if args[0] == symbol.varargslist: names, defaults, flags = self.com_arglist(args[1:]) else: names = defaults = () flags = 0 doc = self.get_docstring(nodelist[-1]) # code for function code = self.com_node(nodelist[-1]) if doc is not None: assert isinstance(code, Stmt) assert isinstance(code.nodes[0], Discard) del code.nodes[0] return Function(decorators, name, names, defaults, flags, doc, code, lineno=lineno)
def com_try_except_finally(self, nodelist): # ('try' ':' suite # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] # | 'finally' ':' suite)) if nodelist[3][0] == token.NAME: # first clause is a finally clause: only try-finally return TryFinally(self.com_node(nodelist[2]), self.com_node(nodelist[5]), lineno=nodelist[0][2]) #tryexcept: [TryNode, [except_clauses], elseNode)] clauses = [] elseNode = None finallyNode = None for i in range(3, len(nodelist), 3): node = nodelist[i] if node[0] == symbol.except_clause: # except_clause: 'except' [expr [(',' | 'as') expr]] */ if len(node) > 2: expr1 = self.com_node(node[2]) if len(node) > 4: expr2 = self.com_assign(node[4], OP_ASSIGN) else: expr2 = None else: expr1 = expr2 = None clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) if node[0] == token.NAME: if node[1] == 'else': elseNode = self.com_node(nodelist[i+2]) elif node[1] == 'finally': finallyNode = self.com_node(nodelist[i+2]) try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, lineno=nodelist[0][2]) if finallyNode: return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) else: return try_except
def get_docs(fileName): """Retrieve information from the parse tree of a source file. fileName Name of the file to read Python source code from. """ source = open(fileName).read() basename = os.path.basename(os.path.splitext(fileName)[0]) ast = parser.suite(source) return ModuleInfo(ast.totuple(), basename)
