From f33273e17ce0498bdd42435c62625a2bec92d50a Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Tue, 30 Apr 2024 16:32:16 +0200 Subject: [PATCH 01/18] Feature: Optionally generating a dotfile for Graphviz etc. This adds a command line flag for giving a filename to write a dotfile to. The dotfile can than be processed further and be plotted by tools like graphviz. --- requirements.txt | 2 ++ src/vis.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f63e1b7..6fb7c8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ matplotlib==3.8.4 networkx==3.3 pyvis==0.3.2 setuptools==69.5.1 +scipy==1.13.0 +pydot==2.0.0 diff --git a/src/vis.py b/src/vis.py index f985717..47c733f 100644 --- a/src/vis.py +++ b/src/vis.py @@ -13,6 +13,7 @@ from matplotlib.colors import hsv_to_rgb from pyvis.network import Network import networkx as nx +from networkx.drawing.nx_pydot import write_dot import graphviz @@ -307,13 +308,20 @@ def get_args(): help="alternate root, if the project root differs from" " the directory that the main script is in", ) + parser.add_argument( + "-d", + "--dot", + dest="dotfile", + type=str, + help="generate dotfile of graph", + ) # TODO implement ability to ignore certain modules # parser.add_argument('-i', '--ignore', dest='ignorefile', type=str, # help='file that contains names of modules to ignore') return parser.parse_args() -def generate_pyvis_visualization(mod_dict): +def generate_pyvis_visualization(mod_dict, dotfile=''): def get_hex_color_of_shade(value): if value < 0 or value > 1: raise ValueError("Input value must be between 0 and 1") @@ -380,6 +388,10 @@ def normaliz_between_n1_1(min, max, val): nx_graph.nodes[node]['size'] = size nx_graph.nodes[node]['color'] = get_hex_color_of_shade(norm_val) + if dotfile: + nx.draw(nx_graph) + write_dot(nx_graph, dotfile) + net = Network(directed=True) net.from_nx(nx_graph) net.show_buttons() @@ -398,6 +410,8 @@ def main(): else: root_dir = args.path mod_dict = get_modules_in_dir(root_dir) + + add_immediate_deps_to_modules(mod_dict) print("Module dependencies:") @@ -413,7 +427,10 @@ def main(): # dag.view() # Creates the pyvis visualization - generate_pyvis_visualization(mod_dict) + if args.dotfile is not None: + generate_pyvis_visualization(mod_dict, dotfile=args.dotfile) + else: + generate_pyvis_visualization(mod_dict) if __name__ == "__main__": main() From 818d3559e7dad3e7926e79778f302124143a92dd Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Tue, 30 Apr 2024 17:20:10 +0200 Subject: [PATCH 02/18] Feature: Add module and callback logic for implementing a module filter logic --- src/filter.py | 6 ++++++ src/vis.py | 28 +++++++++++++--------------- 2 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 src/filter.py diff --git a/src/filter.py b/src/filter.py new file mode 100644 index 0000000..af859a1 --- /dev/null +++ b/src/filter.py @@ -0,0 +1,6 @@ + +def filterfunc(names, top): + return True + +def is_test_module(modulename): + return '.tests.' in modulename diff --git a/src/vis.py b/src/vis.py index 47c733f..71fc317 100644 --- a/src/vis.py +++ b/src/vis.py @@ -8,6 +8,7 @@ import os import platform import sys +import filter from collections import defaultdict from modulefinder import ModuleFinder, Module as MFModule from matplotlib.colors import hsv_to_rgb @@ -72,7 +73,6 @@ def abs_mod_name(module, root_dir): mod_name = ".".join(path_parts) return mod_name - def get_modules_from_file(script, root_dir=None, use_sys_path=False): """ Use ModuleFinder.load_file() to get module imports for the given script. @@ -107,7 +107,6 @@ def get_modules_from_file(script, root_dir=None, use_sys_path=False): return modules - def get_modules_in_dir(root_dir, ignore_venv=True): """ Walk a directory recursively and get the module imports for all .py files in the directory. @@ -130,7 +129,6 @@ def get_modules_in_dir(root_dir, ignore_venv=True): mods[mod_name] = mod return mods - class Module(MFModule, object): """ Extension of modulefinder.ModuleFinder to add custom attrs. """ @@ -141,7 +139,6 @@ def __init__(self, *args, **kwargs): # value = list of names imported from that module self.direct_imports = {} - def _unpack_opargs(code): """ Step through the python bytecode and generate a tuple (int, int, int): (operation_index, operation_byte, argument_byte) for each operation. @@ -170,7 +167,6 @@ def _unpack_opargs(code): yield (i, op, arg) # Python 1? - def scan_opcodes(compiled): """ This function is stolen w/ slight modifications from the standard library @@ -214,8 +210,7 @@ def scan_opcodes(compiled): yield REL_IMPORT, (level, fromlist, names[oparg]) continue - -def get_fq_immediate_deps(all_mods, module): +def get_fq_immediate_deps(all_mods, module, filterfunc=None): """ From a Module, using the module's absolute path, compile the code and then search through it for the imports and get a list of the immediately @@ -239,9 +234,14 @@ def get_fq_immediate_deps(all_mods, module): if op == ABS_IMPORT: names, top = args + if filterfunc is not None: + filter_result = filterfunc(names, top) + else: + filter_result = True if ( not is_std_lib_module(top.split(".")[0], PY_VERSION) or top in all_mods + or filter_result ): if not names: fq_deps[top].append([]) @@ -259,17 +259,15 @@ def get_fq_immediate_deps(all_mods, module): return fq_deps - -def add_immediate_deps_to_modules(mod_dict): +def add_immediate_deps_to_modules(mod_dict, filterfunc=None): """ Take a module dictionary, and add the names of the modules directly imported by each module in the dictionary, and add them to the module's direct_imports. """ for name, module in sorted(mod_dict.items()): - fq_deps = get_fq_immediate_deps(mod_dict, module) + fq_deps = get_fq_immediate_deps(mod_dict, module, filterfunc=filterfunc) module.direct_imports = fq_deps - def mod_dict_to_dag(mod_dict, graph_name): """ Take a module dictionary, and return a graphviz.Digraph object representing the module import relationships. """ @@ -288,7 +286,6 @@ def mod_dict_to_dag(mod_dict, graph_name): dag.edge(name, di, **attrs) return dag - def get_args(): """ Parse and return command line args. """ parser = argparse.ArgumentParser( @@ -320,7 +317,6 @@ def get_args(): # help='file that contains names of modules to ignore') return parser.parse_args() - def generate_pyvis_visualization(mod_dict, dotfile=''): def get_hex_color_of_shade(value): if value < 0 or value > 1: @@ -411,9 +407,11 @@ def main(): root_dir = args.path mod_dict = get_modules_in_dir(root_dir) - + filterfunc=lambda names, top: True + if filter.filterfunc is not None: + filterfunc=filter.filterfunc - add_immediate_deps_to_modules(mod_dict) + add_immediate_deps_to_modules(mod_dict, filterfunc=filterfunc) print("Module dependencies:") for name, module in sorted(mod_dict.items()): print("\n" + name) From 8fe184328d1f623a06907a01040686fdf70cfec3 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 16:06:50 +0200 Subject: [PATCH 03/18] Feature: Add callbacks for filterfunctions to filter either a whole topmodule or specific submodules (or both) The function `add_immediate_deps_to_modules` and `get_fq_immediate_deps` get two additional keyword arguments for handing over callback functions that take module names and return a boolean value advertising if a module should be processed (return true) or not (return false). The lookup for these callback functions happens in a module called modfilter and the callback functions are named `pkgfilterfunc` and `modfilterfunc` by default. If the module is not existent, standard behavior is assumed (no custom filtering). If the module exists but neither `pkgfilterfunc` nor `modfilterfunc` are defined in the module, a notice is logged to stderr at the end. For logging to stderr, a convenience function eprint is introduced. --- src/filter.py | 6 ----- src/modfilter.py | 7 ++++++ src/vis.py | 63 +++++++++++++++++++++++++++++++++--------------- 3 files changed, 50 insertions(+), 26 deletions(-) delete mode 100644 src/filter.py create mode 100644 src/modfilter.py diff --git a/src/filter.py b/src/filter.py deleted file mode 100644 index af859a1..0000000 --- a/src/filter.py +++ /dev/null @@ -1,6 +0,0 @@ - -def filterfunc(names, top): - return True - -def is_test_module(modulename): - return '.tests.' in modulename diff --git a/src/modfilter.py b/src/modfilter.py new file mode 100644 index 0000000..9b584f6 --- /dev/null +++ b/src/modfilter.py @@ -0,0 +1,7 @@ +# example filterfunction for filtering specific module +def modfilterfunc(modname: str, parentname: str) -> bool: + return True + +# filterfunction for filtering specific top module +def pkgfilterfunc(topmodname: str) -> bool: + return True \ No newline at end of file diff --git a/src/vis.py b/src/vis.py index 71fc317..8eac13f 100644 --- a/src/vis.py +++ b/src/vis.py @@ -4,22 +4,26 @@ import argparse import dis +import graphviz +import importlib.util import matplotlib.colors as mc +import networkx as nx import os import platform import sys -import filter from collections import defaultdict +from libinfo import is_std_lib_module from modulefinder import ModuleFinder, Module as MFModule from matplotlib.colors import hsv_to_rgb -from pyvis.network import Network -import networkx as nx from networkx.drawing.nx_pydot import write_dot +from pyvis.network import Network +from typing import Callable -import graphviz - -from libinfo import is_std_lib_module +if importlib.util.find_spec('modfilter', __package__) is not None: + import modfilter +else: + modfilter = None # actual opcodes LOAD_CONST = dis.opmap["LOAD_CONST"] @@ -49,6 +53,10 @@ JAVA_SYSTEM_NAME = "Java" WINDOWS_SYSTEM_NAME = "Windows" +# function for logging (to stderr) +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + def abs_mod_name(module, root_dir): """ From a Module's absolute path, and the root directory, return a string with how that module would be imported from a script in the root @@ -210,7 +218,7 @@ def scan_opcodes(compiled): yield REL_IMPORT, (level, fromlist, names[oparg]) continue -def get_fq_immediate_deps(all_mods, module, filterfunc=None): +def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool]=lambda topmodname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ From a Module, using the module's absolute path, compile the code and then search through it for the imports and get a list of the immediately @@ -234,19 +242,17 @@ def get_fq_immediate_deps(all_mods, module, filterfunc=None): if op == ABS_IMPORT: names, top = args - if filterfunc is not None: - filter_result = filterfunc(names, top) - else: - filter_result = True if ( not is_std_lib_module(top.split(".")[0], PY_VERSION) or top in all_mods - or filter_result + or pkgfilterfunc(top) ): if not names: fq_deps[top].append([]) for name in names: fq_name = top + "." + name + if not modfilterfunc(name, top): + continue if fq_name in all_mods: # just to make sure it's in the dict fq_deps[fq_name].append([]) @@ -259,13 +265,13 @@ def get_fq_immediate_deps(all_mods, module, filterfunc=None): return fq_deps -def add_immediate_deps_to_modules(mod_dict, filterfunc=None): +def add_immediate_deps_to_modules(mod_dict, pkgfilterfunc: Callable[[str], bool]=lambda topname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ Take a module dictionary, and add the names of the modules directly imported by each module in the dictionary, and add them to the module's direct_imports. """ for name, module in sorted(mod_dict.items()): - fq_deps = get_fq_immediate_deps(mod_dict, module, filterfunc=filterfunc) + fq_deps = get_fq_immediate_deps(mod_dict, module, pkgfilterfunc=pkgfilterfunc, modfilterfunc=modfilterfunc) module.direct_imports = fq_deps def mod_dict_to_dag(mod_dict, graph_name): @@ -396,6 +402,8 @@ def normaliz_between_n1_1(min, max, val): def main(): + endnotice = False + args = get_args() if args.path[-3:] == ".py": script = args.path @@ -406,12 +414,24 @@ def main(): else: root_dir = args.path mod_dict = get_modules_in_dir(root_dir) - - filterfunc=lambda names, top: True - if filter.filterfunc is not None: - filterfunc=filter.filterfunc - add_immediate_deps_to_modules(mod_dict, filterfunc=filterfunc) + # check for filterfunction callback to be present, else use stub lambda + if modfilter is None: + add_immediate_deps_to_modules(mod_dict) + else: + hasfunctions = [hasattr(modfilter, "pkgfilterfunc"), hasattr(modfilter, "modfilterfunc")] + match hasfunctions: + case [False, False]: + endnotice = True + add_immediate_deps_to_modules(mod_dict) + case [True, True]: + add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc, modfilterfunc=modfilter.modfilterfunc) + case [True, False]: + add_immediate_deps_to_modules(mod_dict) + add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc) + case [False, True]: + add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.modfilterfunc) + print("Module dependencies:") for name, module in sorted(mod_dict.items()): print("\n" + name) @@ -429,6 +449,9 @@ def main(): generate_pyvis_visualization(mod_dict, dotfile=args.dotfile) else: generate_pyvis_visualization(mod_dict) + + if endnotice: + eprint("Notice: consider adding a filter function (either pkgfilterfunc or modfilterfunc) to modfilter module or removing modfilter module completely") if __name__ == "__main__": - main() + main() \ No newline at end of file From 9c1471843685bd6f241a0dc2575cbd9dcbd4f728 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 16:55:25 +0200 Subject: [PATCH 04/18] Refactor: Add filtering conditions for 100 project --- src/modfilter.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 9b584f6..d3ec104 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,7 +1,16 @@ +from typing import Callable +import sys + # example filterfunction for filtering specific module def modfilterfunc(modname: str, parentname: str) -> bool: - return True + if not is_test_module(parentname): + return True + else: + print("MODNAME: ", parentname, file=sys.stderr) + return False # filterfunction for filtering specific top module -def pkgfilterfunc(topmodname: str) -> bool: - return True \ No newline at end of file +#def pkgfilterfunc(topmodname: str) -> bool: +# return True + +is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname \ No newline at end of file From c7f7b1698eababfcbda164ed7111757da8851138 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 16:55:25 +0200 Subject: [PATCH 05/18] Refactor: Add filtering conditions for 100 project --- src/modfilter.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 9b584f6..0b0151a 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,7 +1,12 @@ +from typing import Callable +import sys + # example filterfunction for filtering specific module def modfilterfunc(modname: str, parentname: str) -> bool: - return True + return not is_test_module(parentname): # filterfunction for filtering specific top module -def pkgfilterfunc(topmodname: str) -> bool: - return True \ No newline at end of file +#def pkgfilterfunc(topmodname: str) -> bool: +# return True + +is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname \ No newline at end of file From 2c6c6a31c5a7006d455c85af4ae7c8f4b757594b Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 17:25:47 +0200 Subject: [PATCH 06/18] Refactor: Only use one callback function for filtering --- src/vis.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/vis.py b/src/vis.py index 8eac13f..2a2e2c2 100644 --- a/src/vis.py +++ b/src/vis.py @@ -218,7 +218,7 @@ def scan_opcodes(compiled): yield REL_IMPORT, (level, fromlist, names[oparg]) continue -def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool]=lambda topmodname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): +def get_fq_immediate_deps(all_mods, module, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ From a Module, using the module's absolute path, compile the code and then search through it for the imports and get a list of the immediately @@ -245,14 +245,16 @@ def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool] if ( not is_std_lib_module(top.split(".")[0], PY_VERSION) or top in all_mods - or pkgfilterfunc(top) + or modfilterfunc("", top) ): if not names: fq_deps[top].append([]) for name in names: fq_name = top + "." + name if not modfilterfunc(name, top): + eprint("EXCLUDE: ", top, "->", name) continue + if fq_name in all_mods: # just to make sure it's in the dict fq_deps[fq_name].append([]) @@ -265,13 +267,13 @@ def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool] return fq_deps -def add_immediate_deps_to_modules(mod_dict, pkgfilterfunc: Callable[[str], bool]=lambda topname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): +def add_immediate_deps_to_modules(mod_dict, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ Take a module dictionary, and add the names of the modules directly imported by each module in the dictionary, and add them to the module's direct_imports. """ for name, module in sorted(mod_dict.items()): - fq_deps = get_fq_immediate_deps(mod_dict, module, pkgfilterfunc=pkgfilterfunc, modfilterfunc=modfilterfunc) + fq_deps = get_fq_immediate_deps(mod_dict, module, modfilterfunc=modfilterfunc) module.direct_imports = fq_deps def mod_dict_to_dag(mod_dict, graph_name): @@ -419,17 +421,11 @@ def main(): if modfilter is None: add_immediate_deps_to_modules(mod_dict) else: - hasfunctions = [hasattr(modfilter, "pkgfilterfunc"), hasattr(modfilter, "modfilterfunc")] - match hasfunctions: - case [False, False]: + match hasattr(modfilter, "modfilterfunc"): + case False: endnotice = True add_immediate_deps_to_modules(mod_dict) - case [True, True]: - add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc, modfilterfunc=modfilter.modfilterfunc) - case [True, False]: - add_immediate_deps_to_modules(mod_dict) - add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc) - case [False, True]: + case True: add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.modfilterfunc) print("Module dependencies:") @@ -449,9 +445,9 @@ def main(): generate_pyvis_visualization(mod_dict, dotfile=args.dotfile) else: generate_pyvis_visualization(mod_dict) - + if endnotice: - eprint("Notice: consider adding a filter function (either pkgfilterfunc or modfilterfunc) to modfilter module or removing modfilter module completely") + eprint("Notice: consider adding a filter function (modfilterfunc) to modfilter module or removing modfilter module completely.") if __name__ == "__main__": main() \ No newline at end of file From c2cec2b8d38242c6ad0c0d3cafd36b78b14dfa1a Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 17:27:09 +0200 Subject: [PATCH 07/18] Add notice to modfilter.py callback funcs to return false to exclude a module from the graph. --- src/modfilter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modfilter.py b/src/modfilter.py index 8eedaa5..fd5fa2c 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -2,10 +2,14 @@ import sys # example filterfunction for filtering specific module +# return false to exclude module def modfilterfunc(modname: str, parentname: str) -> bool: return not is_test_module(parentname) + + # filterfunction for filtering specific top module +# return false to exclude module #def pkgfilterfunc(topmodname: str) -> bool: # return True From f1102598d4860713703bb59e6e382a377b26a8cd Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 17:32:45 +0200 Subject: [PATCH 08/18] Refactor: Only use one callback function instead of two --- src/vis.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/vis.py b/src/vis.py index 8eac13f..2a2e2c2 100644 --- a/src/vis.py +++ b/src/vis.py @@ -218,7 +218,7 @@ def scan_opcodes(compiled): yield REL_IMPORT, (level, fromlist, names[oparg]) continue -def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool]=lambda topmodname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): +def get_fq_immediate_deps(all_mods, module, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ From a Module, using the module's absolute path, compile the code and then search through it for the imports and get a list of the immediately @@ -245,14 +245,16 @@ def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool] if ( not is_std_lib_module(top.split(".")[0], PY_VERSION) or top in all_mods - or pkgfilterfunc(top) + or modfilterfunc("", top) ): if not names: fq_deps[top].append([]) for name in names: fq_name = top + "." + name if not modfilterfunc(name, top): + eprint("EXCLUDE: ", top, "->", name) continue + if fq_name in all_mods: # just to make sure it's in the dict fq_deps[fq_name].append([]) @@ -265,13 +267,13 @@ def get_fq_immediate_deps(all_mods, module, pkgfilterfunc: Callable[[str], bool] return fq_deps -def add_immediate_deps_to_modules(mod_dict, pkgfilterfunc: Callable[[str], bool]=lambda topname: True, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): +def add_immediate_deps_to_modules(mod_dict, modfilterfunc: Callable[[str, str], bool]=lambda name, parentname: True): """ Take a module dictionary, and add the names of the modules directly imported by each module in the dictionary, and add them to the module's direct_imports. """ for name, module in sorted(mod_dict.items()): - fq_deps = get_fq_immediate_deps(mod_dict, module, pkgfilterfunc=pkgfilterfunc, modfilterfunc=modfilterfunc) + fq_deps = get_fq_immediate_deps(mod_dict, module, modfilterfunc=modfilterfunc) module.direct_imports = fq_deps def mod_dict_to_dag(mod_dict, graph_name): @@ -419,17 +421,11 @@ def main(): if modfilter is None: add_immediate_deps_to_modules(mod_dict) else: - hasfunctions = [hasattr(modfilter, "pkgfilterfunc"), hasattr(modfilter, "modfilterfunc")] - match hasfunctions: - case [False, False]: + match hasattr(modfilter, "modfilterfunc"): + case False: endnotice = True add_immediate_deps_to_modules(mod_dict) - case [True, True]: - add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc, modfilterfunc=modfilter.modfilterfunc) - case [True, False]: - add_immediate_deps_to_modules(mod_dict) - add_immediate_deps_to_modules(mod_dict, pkgfilterfunc=modfilter.pkgfilterfunc) - case [False, True]: + case True: add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.modfilterfunc) print("Module dependencies:") @@ -449,9 +445,9 @@ def main(): generate_pyvis_visualization(mod_dict, dotfile=args.dotfile) else: generate_pyvis_visualization(mod_dict) - + if endnotice: - eprint("Notice: consider adding a filter function (either pkgfilterfunc or modfilterfunc) to modfilter module or removing modfilter module completely") + eprint("Notice: consider adding a filter function (modfilterfunc) to modfilter module or removing modfilter module completely.") if __name__ == "__main__": main() \ No newline at end of file From 000b5313f179b366424bf5d667e1dc86f837ea4e Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 17:34:51 +0200 Subject: [PATCH 09/18] Remove the `pkgfilterfunc` from modfilter module and add notice to return False to exclude a module --- src/modfilter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 9b584f6..3c524e0 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,7 +1,4 @@ # example filterfunction for filtering specific module +# return False to exclude a module def modfilterfunc(modname: str, parentname: str) -> bool: return True - -# filterfunction for filtering specific top module -def pkgfilterfunc(topmodname: str) -> bool: - return True \ No newline at end of file From 3599d67380cae8e662b98b79ce0e5af9cf6a9efd Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 18:08:07 +0200 Subject: [PATCH 10/18] Add callback to filter parent modules (that are in the source tree) --- src/modfilter.py | 28 +++++++++++++++++++--------- src/vis.py | 22 +++++++++++++--------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index fd5fa2c..5378c02 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,16 +1,26 @@ -from typing import Callable +from typing import Callable, Dict import sys +# example filterfunction to filter modules that import other modules +def parent_mod_filter_func(mod_dict: Dict) -> Dict: + temp = dict(mod_dict) + for name, _ in mod_dict.items(): + if not parent_filter(name): + del temp[name] + return temp + + + return mod_dict + +def parent_filter(modname: str) -> bool: + return not (is_test_module(modname) or is_logging_module(modname)) + # example filterfunction for filtering specific module # return false to exclude module -def modfilterfunc(modname: str, parentname: str) -> bool: - return not is_test_module(parentname) - +def import_mod_filter_func(modname: str, parentname: str) -> bool: + return not (is_test_module(parentname) or is_logging_module(parentname)) -# filterfunction for filtering specific top module -# return false to exclude module -#def pkgfilterfunc(topmodname: str) -> bool: -# return True +is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname -is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname \ No newline at end of file +is_logging_module: Callable[[str], bool] = lambda modname: 'logging' in modname \ No newline at end of file diff --git a/src/vis.py b/src/vis.py index 2a2e2c2..92fc81f 100644 --- a/src/vis.py +++ b/src/vis.py @@ -245,14 +245,15 @@ def get_fq_immediate_deps(all_mods, module, modfilterfunc: Callable[[str, str], if ( not is_std_lib_module(top.split(".")[0], PY_VERSION) or top in all_mods - or modfilterfunc("", top) + and modfilterfunc("", top) ): if not names: + #eprint("ADDING ", top) fq_deps[top].append([]) for name in names: fq_name = top + "." + name if not modfilterfunc(name, top): - eprint("EXCLUDE: ", top, "->", name) + #eprint("Exclude ", top, "->", name) continue if fq_name in all_mods: @@ -396,11 +397,11 @@ def normaliz_between_n1_1(min, max, val): nx.draw(nx_graph) write_dot(nx_graph, dotfile) - net = Network(directed=True) - net.from_nx(nx_graph) - net.show_buttons() - net.toggle_physics(True) - net.show('mygraph.html', notebook=False) + # net = Network(directed=True) + # net.from_nx(nx_graph) + # net.show_buttons() + # net.toggle_physics(True) + # net.show('mygraph.html', notebook=False) def main(): @@ -421,12 +422,15 @@ def main(): if modfilter is None: add_immediate_deps_to_modules(mod_dict) else: - match hasattr(modfilter, "modfilterfunc"): + match hasattr(modfilter, "parent_mod_filter_func"): + case True: + mod_dict = modfilter.parent_mod_filter_func(mod_dict) + match hasattr(modfilter, "import_mod_filter_func"): case False: endnotice = True add_immediate_deps_to_modules(mod_dict) case True: - add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.modfilterfunc) + add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.import_mod_filter_func) print("Module dependencies:") for name, module in sorted(mod_dict.items()): From 259040d15ea4f9c82ae7f333759998da73b8bec0 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Thu, 2 May 2024 18:19:11 +0200 Subject: [PATCH 11/18] Add convenience eprint function to modfilter (logging to stderr) --- src/modfilter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/modfilter.py b/src/modfilter.py index 5378c02..945a67d 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,11 +1,15 @@ from typing import Callable, Dict import sys +def eprint(content, end="\n"): + print(content, file=sys.stderr, end=end) + # example filterfunction to filter modules that import other modules def parent_mod_filter_func(mod_dict: Dict) -> Dict: temp = dict(mod_dict) for name, _ in mod_dict.items(): if not parent_filter(name): + eprint("Filter: ", name) del temp[name] return temp From 1aca2ded43c235bf60385b28286c90cec25b8639 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Fri, 3 May 2024 06:13:55 +0200 Subject: [PATCH 12/18] Refactor modfilter.py: Remove redundant return statement --- src/modfilter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 945a67d..7a2de1d 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -9,13 +9,9 @@ def parent_mod_filter_func(mod_dict: Dict) -> Dict: temp = dict(mod_dict) for name, _ in mod_dict.items(): if not parent_filter(name): - eprint("Filter: ", name) del temp[name] return temp - - return mod_dict - def parent_filter(modname: str) -> bool: return not (is_test_module(modname) or is_logging_module(modname)) From 94ce0b14aed44fcdf2f74c62d12ec759f4b0f8f3 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Fri, 3 May 2024 06:31:58 +0200 Subject: [PATCH 13/18] Add additional filters (filter django modules) --- src/modfilter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 7a2de1d..6bcc50a 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -13,14 +13,16 @@ def parent_mod_filter_func(mod_dict: Dict) -> Dict: return temp def parent_filter(modname: str) -> bool: - return not (is_test_module(modname) or is_logging_module(modname)) + return not (is_test_module(modname) or is_logging_module(modname) or is_django_module(modname)) # example filterfunction for filtering specific module # return false to exclude module def import_mod_filter_func(modname: str, parentname: str) -> bool: - return not (is_test_module(parentname) or is_logging_module(parentname)) + return not (is_test_module(parentname) or is_logging_module(parentname) or is_django_module(modname)) is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname -is_logging_module: Callable[[str], bool] = lambda modname: 'logging' in modname \ No newline at end of file +is_logging_module: Callable[[str], bool] = lambda modname: 'logging' in modname + +is_django_module: Callable[[str], bool] = lambda modname: 'django' in modname \ No newline at end of file From 0a4a3e3b73e079232992e7145180f5ffa934ebdb Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Fri, 3 May 2024 09:20:13 +0200 Subject: [PATCH 14/18] Refactor vis.py: Fix logical bug in filtering modules (if conditional in `get_fq_immediate_deps`) and remove debug prints. --- src/vis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/vis.py b/src/vis.py index 92fc81f..79e84a0 100644 --- a/src/vis.py +++ b/src/vis.py @@ -243,17 +243,15 @@ def get_fq_immediate_deps(all_mods, module, modfilterfunc: Callable[[str, str], if op == ABS_IMPORT: names, top = args if ( - not is_std_lib_module(top.split(".")[0], PY_VERSION) - or top in all_mods + (not is_std_lib_module(top.split(".")[0], PY_VERSION) + or top in all_mods) and modfilterfunc("", top) ): if not names: - #eprint("ADDING ", top) fq_deps[top].append([]) for name in names: fq_name = top + "." + name if not modfilterfunc(name, top): - #eprint("Exclude ", top, "->", name) continue if fq_name in all_mods: From 4e80f20ec5ac19f19110ddd10bc8289a3038ef78 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Fri, 3 May 2024 09:21:57 +0200 Subject: [PATCH 15/18] Refactor modfilter.py: Fix parameter in `is_django_module` call inside of `import_mod_filter_func` and fix/improve `eprint` convenience function --- src/modfilter.py | 6 +++--- src/vis.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index 6bcc50a..b0c83cc 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -1,8 +1,8 @@ from typing import Callable, Dict import sys -def eprint(content, end="\n"): - print(content, file=sys.stderr, end=end) +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) # example filterfunction to filter modules that import other modules def parent_mod_filter_func(mod_dict: Dict) -> Dict: @@ -18,7 +18,7 @@ def parent_filter(modname: str) -> bool: # example filterfunction for filtering specific module # return false to exclude module def import_mod_filter_func(modname: str, parentname: str) -> bool: - return not (is_test_module(parentname) or is_logging_module(parentname) or is_django_module(modname)) + return not (is_test_module(parentname) or is_logging_module(parentname) or is_django_module(parentname)) is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname diff --git a/src/vis.py b/src/vis.py index 79e84a0..03a97bf 100644 --- a/src/vis.py +++ b/src/vis.py @@ -395,11 +395,11 @@ def normaliz_between_n1_1(min, max, val): nx.draw(nx_graph) write_dot(nx_graph, dotfile) - # net = Network(directed=True) - # net.from_nx(nx_graph) - # net.show_buttons() - # net.toggle_physics(True) - # net.show('mygraph.html', notebook=False) + net = Network(directed=True) + net.from_nx(nx_graph) + net.show_buttons() + net.toggle_physics(True) + net.show('mygraph.html', notebook=False) def main(): From 89e3923f9c45a2bd191bf40dcc4081a1c868f710 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Mon, 6 May 2024 11:25:44 +0200 Subject: [PATCH 16/18] Documentation: Add comments and explanations to filter callbacks --- src/modfilter.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/modfilter.py b/src/modfilter.py index b0c83cc..978a205 100644 --- a/src/modfilter.py +++ b/src/modfilter.py @@ -4,7 +4,7 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -# example filterfunction to filter modules that import other modules +# function to edit mod_dict and filter out modules from project tree (will be parsed for imports) def parent_mod_filter_func(mod_dict: Dict) -> Dict: temp = dict(mod_dict) for name, _ in mod_dict.items(): @@ -12,13 +12,19 @@ def parent_mod_filter_func(mod_dict: Dict) -> Dict: del temp[name] return temp +# example filter function for listed modules in project tree +# return false to exclude module def parent_filter(modname: str) -> bool: - return not (is_test_module(modname) or is_logging_module(modname) or is_django_module(modname)) + # example filter logic + #return not (is_test_module(modname) or is_logging_module(modname) or is_django_module(modname)) + return True # example filterfunction for filtering specific module # return false to exclude module def import_mod_filter_func(modname: str, parentname: str) -> bool: - return not (is_test_module(parentname) or is_logging_module(parentname) or is_django_module(parentname)) + # Example filter logic + #return not (is_test_module(parentname) or is_logging_module(parentname) or is_django_module(parentname)) + return True is_test_module: Callable[[str], bool] = lambda modname: '.tests' in modname From 4f967d45377f43594e98bbc879ee1274cadc7e25 Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Mon, 6 May 2024 11:37:53 +0200 Subject: [PATCH 17/18] Refactor vis.py: Adjust notice, if modfilter.py existent but no callback function implemented. --- src/vis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/vis.py b/src/vis.py index 2a455ac..4004be5 100644 --- a/src/vis.py +++ b/src/vis.py @@ -426,11 +426,14 @@ def main(): mod_dict = modfilter.parent_mod_filter_func(mod_dict) match hasattr(modfilter, "import_mod_filter_func"): case False: - endnotice = True add_immediate_deps_to_modules(mod_dict) case True: add_immediate_deps_to_modules(mod_dict, modfilterfunc=modfilter.import_mod_filter_func) + # print notice to either implement one of the callbacks or consider removing modfilter module + if not hasattr(modfilter, "parent_mod_filter_func") and hasattr(modfilter, "import_mod_filter_func"): + endnotice = True + print("Module dependencies:") for name, module in sorted(mod_dict.items()): print("\n" + name) @@ -450,7 +453,7 @@ def main(): generate_pyvis_visualization(mod_dict) if endnotice: - eprint("Notice: consider adding a filter function (modfilterfunc) to modfilter module or removing modfilter module completely.") + eprint("Notice: consider adding one of the filter functions (parent_mod_filter_func or import_mod_filter_func) to modfilter module or removing modfilter module completely.") if __name__ == "__main__": main() \ No newline at end of file From 13567ed4293c865707a04badba2be1baf911fb5c Mon Sep 17 00:00:00 2001 From: Lucas Haupt Date: Mon, 6 May 2024 11:39:19 +0200 Subject: [PATCH 18/18] Documentation in README.md: Add explanation for custom filter logic --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 5bc551a..e907b14 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ pip install -r requirements.txt ``` # must have venv/venv3 activated $ python src/vis.py +# show help and optional cmd flags +$ python src/vis.py -h ``` __Example:__ @@ -68,3 +70,12 @@ Also displays with `graphviz`: Another example graph from a [slightly more substantial project](https://github.com/nicolashahn/set-solver) (blue arrows/nodes indicate modules where the code does not live in the project directory [such as modules installed through pip]): ![](examples/set-solver.png) + +### Custom filter logic + +If not existent create a module `modfilter.py` alongside `vis.py` in src/ and add one or both of the following two callback functions: +`parent_mod_filter_func(mod_dict: Dict) -> Dict` +`import_mod_filter_func(modname: str, parentname: str) -> bool` + +vis.py checks for a module modfilter.py and these two callback functions as part of the modfilter module and calls them during processing +if existent to allow for custom filtering of modules. A commented example `modfilter.py` is already included in the project.