Python ast 模块，get_docstring() 实例源码

def _get_module_commands(module):
    # type: (ast.Module) -> typing.Generator[_EntryPoint, None, None]
    """Yield all Command objects represented by the python module.

    Module commands consist of a docopt-style module docstring and a callable
    Command class.

    Args:
        module: An ast.Module object used to retrieve docopt-style commands.

    Yields:
        Command objects that represent entry points to append to setup.py.
    """
    cls = next((n for n in module.body
                if isinstance(n, ast.ClassDef) and n.name == 'Command'), None)
    if not cls:
        return
    methods = (n.name for n in cls.body if isinstance(n, ast.FunctionDef))
    if '__call__' not in methods:
        return
    docstring = ast.get_docstring(module)
    for commands, _ in usage.parse_commands(docstring):
        yield _EntryPoint(commands[0], next(iter(commands[1:]), None), None)

def _get_class_commands(module):
    # type: (ast.Module) -> typing.Generator[_EntryPoint, None, None]
    """Yield all Command objects represented by python classes in the module.

    Class commands are detected by inspecting all callable classes in the
    module for docopt-style docstrings.

    Args:
        module: An ast.Module object used to retrieve docopt-style commands.

    Yields:
        Command objects that represent entry points to append to setup.py.
    """
    nodes = (n for n in module.body if isinstance(n, ast.ClassDef))
    for cls in nodes:
        methods = (n.name for n in cls.body if isinstance(n, ast.FunctionDef))
        if '__call__' in methods:
            docstring = ast.get_docstring(cls)
            for commands, _ in usage.parse_commands(docstring):
                yield _EntryPoint(commands[0], next(iter(commands[1:]), None),
                                  cls.name)

def _get_function_commands(module):
    # type: (ast.Module) -> typing.Generator[_EntryPoint, None, None]
    """Yield all Command objects represented by python functions in the module.

    Function commands consist of all top-level functions that contain
    docopt-style docstrings.

    Args:
        module: An ast.Module object used to retrieve docopt-style commands.

    Yields:
        Command objects that represent entry points to append to setup.py.
    """
    nodes = (n for n in module.body if isinstance(n, ast.FunctionDef))
    for func in nodes:
        docstring = ast.get_docstring(func)
        for commands, _ in usage.parse_commands(docstring):
            yield _EntryPoint(commands[0], next(iter(commands[1:]), None),
                              func.name)

def get_function_markdown(node):
    """Parse a function node."""
    output = ""
    output += "#### " + node.name + "("
    if len(node.args.args) > 0:
        output += ", ".join(
            str(x.arg) for x in node.args.args)
    output += ')'
    output += '\n'
    output += '\n'

    doc_string = ast.get_docstring(node)
    if doc_string:
        output += "> " + doc_string + ""
    else:
        output += "> No docstring found."
    output += '\n'
    output += '\n'

    return output

def visit_Module(self, node):
        obj = {
            'type': 'module',
            'docstring': self.get_docstring(node),
            'classes': [],
            'functions': [],
            'variables': [],
            'imports': [],
        }

        for node in imap(self.visit, node.body):
            if node['type'] == 'function':
                obj['functions'].append(node)
            elif node['type'] == 'class':
                obj['classes'].append(node)
            elif node['type'] == 'assign':
                obj['variables'].append(node)
            elif node['type'] in ('import', 'import_from'):
                obj['imports'].append(node)
            elif node['type'] in ('expr', ):
                continue

        return obj

def visit_ClassDef(self, node):
        obj = {
            'type': 'class',
            'name': node.name,
            'docstring': self.get_docstring(node),
            'bases': list(ifilter(lambda k: k.get('name') != 'object', [
                {'name': i.id} if isinstance(i, ast.Name) else self.visit(i) for i in node.bases
            ])),
            'attributes': [],
            'functions': [],
        }

        for node in imap(self.visit, node.body):
            if node['type'] == 'function':
                obj['functions'].append(node)
            elif node['type'] == 'assign':
                obj['attributes'].append(node)

        return obj

def module_from_ast(self, module_name, filename, t):
        code = self.compiler.code_for_module(module_name, filename, t)
        module = types.ModuleType(module_name, ast.get_docstring(t))
        self.interpreter.run(code, module.__dict__, None)
        return module

def module_from_ast(module_name, filename, t):
    code = compiler.code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    byterun.interpreter.run(code, module.__dict__, None)
    return module

def compile_function(self, t):
        self.load_const(ast.get_docstring(t))
        for arg in t.args.args:
            self.varnames[arg.arg]
        if t.args.vararg: self.varnames[t.args.vararg.arg]
        if t.args.kwarg:  self.varnames[t.args.kwarg.arg]
        assembly = self(t.body) + self.load_const(None) + op.RETURN_VALUE
        return self.make_code(assembly, t.name,
                              len(t.args.args), t.args.vararg, t.args.kwarg)

def compile_class(self, t):
        docstring = ast.get_docstring(t)
        assembly = (  self.load('__name__')      + self.store('__module__')
                    + self.load_const(t.name)    + self.store('__qualname__')
                    + (no_op if docstring is None else
                       self.load_const(docstring) + self.store('__doc__'))
                    + self(t.body)
                    + self.load_const(None) + op.RETURN_VALUE)
        return self.make_code(assembly, t.name, 0, False, False)

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def compile_function(self, t):
        self.load_const(ast.get_docstring(t))
        for arg in t.args.args:
            self.varnames[arg.arg]
        assembly = self(t.body) + self.load_const(None) + op.RETURN_VALUE
        return self.make_code(assembly, t.name, len(t.args.args))

def compile_class(self, t):
        docstring = ast.get_docstring(t)
        assembly = (  self.load('__name__')      + self.store('__module__')
                    + self.load_const(t.name)    + self.store('__qualname__')
                    + (no_op if docstring is None else
                       self.load_const(docstring) + self.store('__doc__'))
                    + self(t.body) + self.load_const(None) + op.RETURN_VALUE)
        return self.make_code(assembly, t.name, 0)

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def compile_class(self, t):
        docstring = ast.get_docstring(t)
        assembly = (  self.load('__name__')      + self.store('__module__')
                    + self.load_const(t.name)    + self.store('__qualname__')
                    + (no_op if docstring is None else
                       self.load_const(docstring) + self.store('__doc__'))
                    + self(t.body) + self.load_const(None) + op.RETURN_VALUE)
        return self.make_code(assembly, t.name, 0)

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def compile_function(self, t):
        self.load_const(ast.get_docstring(t))
        for arg in t.args.args:
            self.varnames[arg.arg]
        assembly = self(t.body) + self.load_const(None) + op.RETURN_VALUE
        return self.make_code(assembly, t.name, len(t.args.args))

def compile_class(self, t):
        docstring = ast.get_docstring(t)
        assembly = (  self.load('__name__')      + self.store('__module__')
                    + self.load_const(t.name)    + self.store('__qualname__')
                    + (no_op if docstring is None else
                       self.load_const(docstring) + self.store('__doc__'))
                    + self(t.body) + self.load_const(None) + op.RETURN_VALUE)
        return self.make_code(assembly, t.name, 0)

def module_from_ast(module_name, filename, t):
    code = code_for_module(module_name, filename, t)
    module = types.ModuleType(module_name, ast.get_docstring(t))
    exec(code, module.__dict__)
    return module

def visit_FunctionDef(self, node, is_async=False):
        docstring = f"\"{get_docstring(node, True)}\""
        body = node.body
        if docstring:
            body = body[1:]  # Don't mention it

        summary = ""\
            + f"{interpret_async(is_async)} function called \"{node.name}\""\
            + f", taking {self.visit(node.args)}"\
            + (f", and returning a value of {self.visit(node.returns)}" if node.returns else "")\
            + (f", with the docstring of {docstring}" if docstring else "")\
            + f", with a body of {self.visit(body)}"

        return summary

def find_info(*path_parts):
    finder = MetaDataFinder()
    node = ast.parse(read(*path_parts).encode('utf-8'))
    finder.visit(node)
    info = finder.data
    info['docstring'] = ast.get_docstring(node)
    return info

def class_check(node, kw=False):
    '''
    Class specific check
        Action - check private method, move to utils
        check docstring
        scan for class - recursive class check
        scan for function - function check
    '''
    status = True

    for child in ast.iter_child_nodes(node):
        if isinstance(child, ast.FunctionDef):
            if kw and child.name.startswith("_") and child.name != "__init__":
                print node.name, child.name, "should move to utils"
                status = False
            tmp_status = func_check(child, kw)
            status &= tmp_status
        elif isinstance(child, ast.ClassDef):
            tmp_status = class_check(child, kw)
            status &= tmp_status

    if ast.get_docstring(node) is None:
        # check for docstring
        print node.name, "doesn't contain any docstring"
        status = False
    return status

def visit_FunctionDef(self, node):
        name = node.name
        args = self.visit(node.args)
        body = [self.visit(x) for x in node.body]
        docstring = ast.get_docstring(node)
        if docstring:
            body = body[1:]
        if hasattr(node, "returns"):
            returns = self.visit(node.returns)
        else:
            returns = None
        # TODO: decorator_list
        return cpp.FunctionDef(name=name, args=args, body=body, docstring=docstring, returns=returns)

def visit_ClassDef(self, node):
        name = node.name
        bases = [self.visit(x) for x in node.bases]
        if six.PY3:
            keywords = [self.visit(x) for x in node.keywords]
        body = [self.visit(x) for x in node.body]
        docstring = ast.get_docstring(node)
        if docstring:
            body = body[1:]
        # TODO: decorator_list
        if six.PY3:
            return cpp.ClassDef(name=name, bases=bases, keywords=keywords, body=body, docstring=docstring)
        else:
            return cpp.ClassDef(name=name, bases=bases, body=body, docstring=docstring)

def coerce_file(fn):
    """Coerce content of given file to something useful for setup(), turn :
       .py into mock object with description and version fields,
       .md into rst text. Remove images with "nopypi" alt text along the way.

       :url: https://github.com/Kraymer/setupgoon
    """
    import ast
    import os
    import re
    import subprocess  # noqa
    text = open(os.path.join(os.path.dirname(__file__), fn)).read()
    if fn.endswith('.py'):  # extract version, docstring etc out of python file
        mock = type('mock', (object,), {})()
        for attr in ('version', 'author', 'author_email', 'license'):
            regex = r'^__%s__\s*=\s*[\'"]([^\'"]*)[\'"]$' % attr
            m = re.search(regex, text, re.MULTILINE)
            setattr(mock, attr, m.group(1) if m else None)
        mock.docstring = ast.get_docstring(ast.parse(text))
        return mock
    if fn.endswith('md') and 'upload' in sys.argv:  # convert md to rest on pypi package upload
        text = '\n'.join([l for l in text.split('\n') if '![nopypi' not in l])
        p = subprocess.Popen(['pandoc', '-t', 'rst'], stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE)
        text, stderr = p.communicate(text)
    return text

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def _add_crc(self, node, kind, name):
        self.current_crc = CRC(name, kind)
        docstring = ast.get_docstring(node) or ""
        for line in docstring.split('\n'):
            self.get_collaborator(line)
            self.get_responsibility(line)
        self._crcs.append(self.current_crc)

def get_class_markdown(node):
    """Parse a class node."""
    output = ""
    if node.name.lower().strip() != "meta":
        output += '--------------------'
        output += '\n'
    output += "## " + node.name
    output += '\n'
    doc_string = ast.get_docstring(node)
    if doc_string:
        output += "" + doc_string + ""
    output += '\n'
    output += '\n'

    return output

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def coerce_file(fn):
    """Coerce file content to something useful for setuptools.setup(), turn :
       .py into mock object by extracting __special__ attributes values
       .md into rst text. Remove images with "[nopypi" alt text and emojis
       :url: https://github.com/Kraymer/setupgoon
    """
    import ast
    import os
    import re
    import subprocess
    import tempfile
    import time  # noqa
    text = open(os.path.join(os.path.dirname(__file__), fn)).read()
    if fn.endswith('.py'):  # extract version, docstring etc out of python file
        mock = type('mock', (object,), {})()
        for attr in ('version', 'author', 'author_email', 'license', 'url'):
            regex = r'^__%s__\s*=\s*[\'"]([^\'"]*)[\'"]$' % attr
            m = re.search(regex, text, re.MULTILINE)
            setattr(mock, attr, m.group(1) if m else None)
        mock.docstring = ast.get_docstring(ast.parse(text))
        if mock.version.endswith('dev'):
            mock.version += str(int(time.time()))
        return mock
    if 'upload' in sys.argv and fn.endswith('md'):  # convert markdown to rest
        text = '\n'.join([l for l in text.split('\n') if '[nopypi' not in l])
        text = re.sub(r':\S+:', '', text)  # no emojis
        with tempfile.NamedTemporaryFile(mode='w+') as tmp:
            tmp.write(text)
            tmp.flush()
            text, stderr = subprocess.Popen(['pandoc', '-t', 'rst', tmp.name],
                stdout=subprocess.PIPE).communicate()
    return text.decode('utf-8')

def _add_file_docs(self, tree):
    """Extracts the file docstring of the .bzl file."""
    docstring = ast.get_docstring(tree)
    if docstring == None:
      return
    lines = docstring.split("\n")
    i = 0
    for line in lines:
      if line != '':
        i = i + 1
      else:
        break

    self.title = " ".join(lines[:i])
    self.description = "\n".join(lines[i + 1:])

def _add_macro_doc(self, stmt):
    # The defaults array contains default values for the last arguments.
    # The first shift arguments are mandatory.
    shift = len(stmt.args.args) - len(stmt.args.defaults)

    rule = self.__language.rule.add()
    rule.name = stmt.name
    rule.type = build_pb2.RuleDefinition.MACRO

    doc = ast.get_docstring(stmt)
    if doc:
      extracted_docs = common.parse_docstring(doc)
      rule.documentation = extracted_docs.doc
      if extracted_docs.example_doc:
        rule.example_documentation = extracted_docs.example_doc
    else:
      extracted_docs = common.ExtractedDocs(
          doc="", attr_docs={}, example_doc="", output_docs={})

    for i in range(len(stmt.args.args)):
      attr = rule.attribute.add()
      attr_name = stmt.args.args[i].id
      attr.name = attr_name

      if attr_name in extracted_docs.attr_docs:
        attr.documentation = extracted_docs.attr_docs[attr_name]

      if i < shift:  # The first arguments are mandatory
        attr.mandatory = True
        attr.type = build_pb2.Attribute.UNKNOWN
      else:
        node = stmt.args.defaults[i - shift]
        attr.mandatory = False
        attr.type = get_type(node)
        if attr.type == build_pb2.Attribute.BOOLEAN:
          attr.default = node.id

    for template, doc in extracted_docs.output_docs.iteritems():
      output = rule.output.add()
      output.template = template
      output.documentation = doc

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def visit_Module(self, node):
        if sys.version_info >= (3, 7):
            docstring = ast.get_docstring(node, clean=False)
            for t in self.check_str(docstring):
                yield t
        for t in self.generic_visit(node):
            yield t

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def _get_docstring(self, node):
        return ast.get_docstring(node)

def cmp(self, node, pattern):
        docstring = ast.get_docstring(node)
        if docstring is None:
            return False
        LOG.debug("Comparing %r and %r", docstring, pattern.pattern)
        return pattern(docstring)

def get_doc(self, module):
        for node in module.body:
            if isinstance(node, ast.ClassDef):
                yield ast.get_docstring(node)
                for sub_node in node.body:
                    if isinstance(sub_node, ast.FunctionDef):
                        yield ast.get_docstring(sub_node)

def test_get_docstring(self):
        node = ast.parse('def foo():\n  """line one\n  line two"""')
        self.assertEqual(ast.get_docstring(node.body[0]),
                         'line one\nline two')

def rst_sections_in_module(tree, module_name, exports):
    docstring = ast.get_docstring(tree)
    if docstring:
        yield docstring
    for toplevel in tree.body:
        try:
            if toplevel.name not in exports:
                continue
        except AttributeError:
            continue

        print(
            'Documenting {}.{}'.format(module_name, toplevel.name),
            file=sys.stderr)
        class_doc = ast.get_docstring(toplevel)
        if class_doc:
            yield toplevel.name + '\n' + '-' * len(toplevel.name)
            yield class_doc
            if type(toplevel) == ast.ClassDef:
                for fun in toplevel.body:
                    if type(fun) != ast.FunctionDef:
                        continue
                    fun_doc = ast.get_docstring(fun)
                    if fun_doc:
                        fun_sig = signature(module_name, toplevel.name, fun)
                        yield fun.name + '\n' + '+' * len(fun.name)
                        yield 'Signature:\n    ' + fun_sig
                        yield fun_doc
                    else:
                        if not fun.name.startswith('_'):
                            print('Missing docstring for ' + fun.name, file=sys.stderr)

def get_docstring(self, node):
        return self.parser.parse(ast.get_docstring(node))

def visit_FunctionDef(self, node):
        return {
            'type': 'function',
            'docstring': self.get_docstring(node),
            'decorators': [],
            'name': node.name,
            'args': self.visit(node.args)
        }

def func_check(node, kw=False):
    '''
    Function specific check
        Action - check return type
        Action - check pstep/psubstep buddy with report_step/report_substep_status
        check docstring
        check print statement - should use print_utils
        scan for class - class check
        scan for function - recursive function check
    '''
    status = True
    have_return = False
    substep_count = 0

    # investigate everything in a function
    for child in ast.walk(node):
        if child != node and isinstance(child, ast.FunctionDef):
            # check for private method in action file
            if kw and child.name.startswith("_"):
                print node.name, child.name, "should move to utils"
                status = False
            tmp_status = func_check(child, kw)
            status &= tmp_status
        elif child != node and isinstance(child, ast.ClassDef):
            tmp_status = class_check(child, kw)
            status &= tmp_status
        elif 'war_print_class.py' not in sys.argv[1] and isinstance(child, ast.Print):
            # check for print statement
            status = False
            print "Please use print_Utils instead of print in {}: {}".format(
                                                    sys.argv[1], child.lineno)
        elif isinstance(child, ast.Return):
            # check for return statement
            have_return = True
        elif isinstance(child, ast.Attribute) and child.attr == 'pSubStep':
            # check for Substep and report substep pair
            substep_count += 1
        elif isinstance(child, ast.Attribute) and child.attr == 'report_substep_status':
            substep_count -= 1

    if ast.get_docstring(node) is None:
        print node.name, "doesn't contain any docstring"
        status = False
    if kw and not have_return and node.name != "__init__":
        print node.name, "doesn't contain a return statement"
        status = False
    if kw and substep_count:
        print node.name, "have non-pair pSubStepreport_substep_status"
        status = False
    return status

def extract_info(self, module):
        """
        .. py:attribute:: extract_info()

           This function loop over parser object and extracts the informations
           (contains name, documentation, line number etc.) for function, class
           and attribute objects.

           :rtype: dict

        .. note::

        """
        def extracter(root_nod):
            for node in root_nod.body:
                if isinstance(node, ast.ClassDef):
                    yield {
                        "name": node.name,
                        "lineno": node.lineno - 1,
                        "docstring": ast.get_docstring(node),
                        "type": 'class'}
                    for sub_node in node.body:
                        args_ = []
                        if isinstance(sub_node, ast.FunctionDef):
                            for arg in sub_node.args.args:
                                    try:
                                        args_.append(arg.id)
                                    except AttributeError:
                                        args_.extend([item.id for item in arg.elts if item.id != 'self'])

                            yield {
                                "name": sub_node.name,
                                "lineno": sub_node.lineno - 1,
                                "docstring": ast.get_docstring(sub_node),
                                "type": 'attribute',
                                "args": args_,
                                "header": ''}
                            for n in extracter(sub_node):
                                yield n
                elif isinstance(node, ast.FunctionDef):
                    args_ = []
                    for arg in node.args.args:
                        try:
                            args_.append(arg.id)
                        except AttributeError:
                            args_.extend([item.id for item in arg])
                    yield {
                        "name": node.name,
                        "lineno": node.lineno - 1,
                        "docstring": ast.get_docstring(node),
                        "type": 'function',
                        "args": args_}
                    for n in extracter(node):
                        yield n
        return extracter(module)