From 94a03d73a35a830c56edd15eeeaa9868eb3636d3 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Fri, 17 Jul 2020 13:39:57 +0200 Subject: [PATCH] Python: CG trace: blackify And make code pass flake8 tests --- .../tools/recorded-call-graph-metrics/.flake8 | 6 ++ .../recorded-call-graph-metrics/cg_trace.py | 92 ++++++++++--------- .../requirements.txt | 5 + 3 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 python/tools/recorded-call-graph-metrics/.flake8 create mode 100644 python/tools/recorded-call-graph-metrics/requirements.txt diff --git a/python/tools/recorded-call-graph-metrics/.flake8 b/python/tools/recorded-call-graph-metrics/.flake8 new file mode 100644 index 00000000000..0d808caa1ea --- /dev/null +++ b/python/tools/recorded-call-graph-metrics/.flake8 @@ -0,0 +1,6 @@ +# As described in https://github.com/psf/black/blob/master/docs/compatible_configs.md#flake8 +# and https://black.readthedocs.io/en/stable/the_black_code_style.html#line-length +[flake8] +max-line-length = 88 +select = C,E,F,W,B,B950 +ignore = E203, E501, W503 diff --git a/python/tools/recorded-call-graph-metrics/cg_trace.py b/python/tools/recorded-call-graph-metrics/cg_trace.py index 55d6f6e4665..8d81f6bcec6 100755 --- a/python/tools/recorded-call-graph-metrics/cg_trace.py +++ b/python/tools/recorded-call-graph-metrics/cg_trace.py @@ -21,7 +21,6 @@ be able to be called in a place where it is not? Currently not something we're l # - https://github.com/gak/pycallgraph (display call-graph with graphviz after python execution) import argparse -import bdb from io import StringIO import sys import os @@ -34,11 +33,15 @@ from typing import Optional # copy-paste For interactive ipython sessions # import IPython; sys.stdout = sys.__stdout__; IPython.embed(); sys.exit() + def debug_print(*args, **kwargs): # print(*args, **kwargs, file=sys.__stderr__) pass + _canonic_filename_cache = dict() + + def canonic_filename(filename): """Return canonical form of filename. (same as Bdb.canonic) @@ -58,9 +61,10 @@ def canonic_filename(filename): @dataclasses.dataclass(frozen=True, eq=True, order=True) -class Call(): +class Call: """A call """ + filename: str linenum: int inst_index: int @@ -74,9 +78,9 @@ class Call(): debug_print(b.dis()) return cls( - filename = canonic_filename(code.co_filename), - linenum = frame.f_lineno, - inst_index = frame.f_lasti, + filename=canonic_filename(code.co_filename), + linenum=frame.f_lineno, + inst_index=frame.f_lasti, ) @@ -85,15 +89,23 @@ def better_compare_for_dataclass(cls): objects of the same class. This decorator extends the functionality to compare class name if used against other objects. """ - for op in ['__lt__', '__le__', '__gt__', '__ge__',]: + for op in [ + "__lt__", + "__le__", + "__gt__", + "__ge__", + ]: old = getattr(cls, op) + def new(self, other): if type(self) == type(other): return old(self, other) return getattr(str, op)(self.__class__.__name__, other.__class__.__name__) + setattr(cls, op, new) return cls + @dataclasses.dataclass(frozen=True, eq=True, order=True) class Callee: pass @@ -115,12 +127,12 @@ class ExternalCallee(Callee): @classmethod def from_arg(cls, func): # if func.__name__ == "append": - # import IPython; sys.stdout = sys.__stdout__; IPython.embed(); sys.exit() + # import IPython; sys.stdout = sys.__stdout__; IPython.embed(); sys.exit() return cls( module=func.__module__, qualname=func.__qualname__, - is_builtin=type(func) == BUILTIN_FUNCTION_OR_METHOD + is_builtin=type(func) == BUILTIN_FUNCTION_OR_METHOD, ) @@ -132,6 +144,7 @@ class PythonCallee(Callee): should (hopefully) be uniquely identified by its name and location (filename+line number) """ + filename: str linenum: int funcname: str @@ -140,9 +153,9 @@ class PythonCallee(Callee): def from_frame(cls, frame): code = frame.f_code return cls( - filename = canonic_filename(code.co_filename), - linenum = frame.f_lineno, - funcname = code.co_name, + filename=canonic_filename(code.co_filename), + linenum=frame.f_lineno, + funcname=code.co_name, ) @@ -186,7 +199,8 @@ class CallGraphTracer: sys.setprofile(None) def profilefunc(self, frame, event, arg): - # ignore everything until the first call, since that is `exec` from the `run` method above + # ignore everything until the first call, since that is `exec` from the `run` + # method above if not self.exec_call_seen: if event == "call": self.exec_call_seen = True @@ -211,13 +225,13 @@ class CallGraphTracer: callee = PythonCallee.from_frame(frame) if event == "c_call": - # in c_call, the `frame` argument is frame where the call happens, and the `arg` argument - # is the C function object. + # in c_call, the `frame` argument is frame where the call happens, and the + # `arg` argument is the C function object. call = Call.from_frame(frame) callee = ExternalCallee.from_arg(arg) - debug_print(f'{call} --> {callee}') - debug_print('\n'*5) + debug_print(f"{call} --> {callee}") + debug_print("\n" * 5) self.recorded_calls.add((call, callee)) @@ -227,7 +241,6 @@ class CallGraphTracer: class Exporter: - @staticmethod def export(recorded_calls, outfile_path): raise NotImplementedError() @@ -240,15 +253,14 @@ class Exporter: class CSVExporter(Exporter): - @staticmethod def export(recorded_calls, outfile_path): - with open(outfile_path, 'w', newline='') as csv_file: + with open(outfile_path, "w", newline="") as csv_file: writer = None for (call, callee) in sorted(recorded_calls): data = { **Exporter.dataclass_to_dict(call), - **Exporter.dataclass_to_dict(callee) + **Exporter.dataclass_to_dict(callee), } if writer is None: @@ -257,32 +269,30 @@ class CSVExporter(Exporter): writer.writerow(data) - - print(f'output written to {outfile_path}') + print(f"output written to {outfile_path}") # embed(); sys.exit() class XMLExporter(Exporter): - @staticmethod def export(recorded_calls, outfile_path): - root = etree.Element('root') + root = etree.Element("root") for (call, callee) in sorted(recorded_calls): data = { **Exporter.dataclass_to_dict(call), - **Exporter.dataclass_to_dict(callee) + **Exporter.dataclass_to_dict(callee), } - rc = etree.SubElement(root, 'recorded_call') + rc = etree.SubElement(root, "recorded_call") for k, v in data.items(): # xml library only supports serializing attributes that have string values rc.set(k, str(v)) tree = etree.ElementTree(root) - tree.write(outfile_path, encoding='utf-8', pretty_print=True) + tree.write(outfile_path, encoding="utf-8", pretty_print=True) ################################################################################ @@ -293,13 +303,13 @@ class XMLExporter(Exporter): if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("--csv") + parser.add_argument("--xml") - parser.add_argument('--csv') - parser.add_argument('--xml') - - parser.add_argument('progname', help='file to run as main program') - parser.add_argument('arguments', nargs=argparse.REMAINDER, - help='arguments to the program') + parser.add_argument("progname", help="file to run as main program") + parser.add_argument( + "arguments", nargs=argparse.REMAINDER, help="arguments to the program" + ) opts = parser.parse_args() @@ -309,14 +319,14 @@ if __name__ == "__main__": sys.path[0] = os.path.dirname(opts.progname) with open(opts.progname) as fp: - code = compile(fp.read(), opts.progname, 'exec') + code = compile(fp.read(), opts.progname, "exec") # try to emulate __main__ namespace as much as possible globs = { - '__file__': opts.progname, - '__name__': '__main__', - '__package__': None, - '__cached__': None, + "__file__": opts.progname, + "__name__": "__main__", + "__package__": None, + "__cached__": None, } real_stdout = sys.stdout @@ -334,7 +344,7 @@ if __name__ == "__main__": XMLExporter.export(cgt.recorded_calls, opts.xml) else: for (call, callee) in sorted(cgt.recorded_calls): - print(f'{call} --> {callee}') + print(f"{call} --> {callee}") - print('--- captured stdout ---') - print(captured_stdout.getvalue(), end='') + print("--- captured stdout ---") + print(captured_stdout.getvalue(), end="") diff --git a/python/tools/recorded-call-graph-metrics/requirements.txt b/python/tools/recorded-call-graph-metrics/requirements.txt new file mode 100644 index 00000000000..06e58e23bf2 --- /dev/null +++ b/python/tools/recorded-call-graph-metrics/requirements.txt @@ -0,0 +1,5 @@ +lxml +# dev +black +flake8 +flake8-bugbear