From 8ef9dd94ee51ba006571bff26e2fe8fd35db8821 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Thu, 11 Sep 2025 18:09:07 +0200 Subject: [PATCH] add record descriptor info to plugin doc pages --- .gitignore | 14 +++- docs/Makefile | 1 + docs/source/_ext/dissect_plugins.py | 105 ++++++++++++++++++++++++---- docs/source/plugins/index.rst | 1 + pyproject.toml | 88 +++++++++++++++++++++++ requirements.txt | 18 ----- tox.ini | 5 +- 7 files changed, 193 insertions(+), 39 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index d18c6d2..20b295b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,15 @@ +coverage.xml +.coverage +dist/ +.eggs/ +*.egg-info/ +*.pyc +__pycache__/ +.pytest_cache/ +tests/_docs/api +tests/_docs/build +.tox/ + build/ _build/ -__pycache__ - docs/source/api/*/ diff --git a/docs/Makefile b/docs/Makefile index e26386f..31096af 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,6 +17,7 @@ help: clean: Makefile rm -rf source/api/*/ + rm -rf source/plugins/* @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) watch: Makefile diff --git a/docs/source/_ext/dissect_plugins.py b/docs/source/_ext/dissect_plugins.py index b3c7c80..46485e5 100644 --- a/docs/source/_ext/dissect_plugins.py +++ b/docs/source/_ext/dissect_plugins.py @@ -1,5 +1,8 @@ +from __future__ import annotations + +import textwrap from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from dissect.target.exceptions import PluginError from dissect.target.helpers import docs @@ -9,6 +12,9 @@ from sphinx.util.display import status_iterator from sphinx.util.logging import getLogger +if TYPE_CHECKING: + from flow.record import RecordDescriptor + LOGGER = getLogger(__name__) PAGE_TEMPLATE = """.. @@ -24,30 +30,54 @@ """ VARIANT_TEMPLATE = """ -.. list-table:: Details +.. list-table:: :widths: 20 80 * - Module - ``{module}.{class}`` * - Output - ``{output}`` + * - Source + - {source} -**Module documentation** +Module documentation +-------------------- {class_doc} -**Function documentation** +Function documentation +---------------------- {func_doc} + +Record output +------------- + +{record_doc} """ NAMESPACE_TEMPLATE = """ This is a namespace plugin. This means that by running this plugin, it will automatically run all other plugins under this namespace: +.. toctree:: + :maxdepth: 1 + :glob: + {exports} """ +INDEX_TEMPLATE = """\ +Plugin Reference +================ + +.. toctree:: + :maxdepth: 1 + :glob: + + /plugins/*/index + /plugins/* +""" def builder_inited(app: Sphinx) -> None: dst = Path(app.srcdir).joinpath("plugins") @@ -58,9 +88,13 @@ def builder_inited(app: Sphinx) -> None: for plugin in plugins(): # Ignore all modules in general as those are all internal or utility - if plugin.module.startswith("general."): + if plugin.path.startswith("general."): continue + # # Exclude plugins without any exports and/or InternalPlugins + # if not plugin.findable or not plugin.exports: + # continue + if ns := plugin.namespace: plugin_map.setdefault(ns, []).append(plugin) @@ -72,17 +106,24 @@ def builder_inited(app: Sphinx) -> None: export = f"{ns}.{export}" plugin_map.setdefault(export, []).append(plugin) + dst.joinpath("index.rst").write_text(INDEX_TEMPLATE) + for name, plugin in status_iterator( plugin_map.items(), colorize("bold", "[Dissect] Writing plugin files... "), length=len(plugin_map), stringify_func=(lambda x: x[0]), ): - dst_path = dst.joinpath(name + ".rst") - if dst_path.exists(): - continue - dst_path.write_text(_format_template(name, plugin)) + if name == plugin[0].namespace: + dst_path = dst.joinpath(f"{name.replace('.', '/')}/index.rst") + else: + dst_path = dst.joinpath(name.replace(".", "/") + ".rst") + + if not dst_path.parent.is_dir(): + dst_path.parent.mkdir() + + dst_path.write_text(_format_template(app, name, plugin)) def build_finished(app: Sphinx, exception: Exception) -> None: @@ -91,14 +132,14 @@ def build_finished(app: Sphinx, exception: Exception) -> None: if app.verbosity > 1: LOGGER.info(colorize("bold", "[Dissect] ") + colorize("darkgreen", "Cleaning generated .rst files")) - for rst in dst.glob("*.rst"): - with open(rst, "rb") as fh: + for rst in dst.glob("**.rst"): + with rst.open("rb") as fh: if fh.read(16) != b"..\n generated": continue rst.unlink() -def _format_template(name: str, plugins: list[dict]) -> str: +def _format_template(app: Sphinx, name: str, plugins: list[dict]) -> str: func_name = name.split(".")[-1] if "." in name else name variants = [] @@ -117,12 +158,14 @@ def _format_template(name: str, plugins: list[dict]) -> str: func_output = "records" func_doc = NAMESPACE_TEMPLATE.format( exports="\n".join( - f"- :doc:`/plugins/{ns}.{export}`" for export in plugin.exports if export != "__call__" + f" /plugins/{ns.replace('.', '/')}/{export}.rst" for export in plugin.exports if export != "__call__" ) ) + record_doc = get_record_docs(plugin.cls.__record_descriptors__) if hasattr(plugin.cls, "__record_descriptors__") else "" else: func = getattr(plugin_class, func_name) func_output, func_doc = docs._get_func_details(func) + record_doc = get_record_docs(func.__record__) if hasattr(func, "__record__") else "This plugin does not generate record output." info = { "module": plugin.module, @@ -130,22 +173,54 @@ def _format_template(name: str, plugins: list[dict]) -> str: "output": func_output, "class_doc": class_doc, "func_doc": func_doc, + "record_doc": record_doc, + "source": f"{app.config.dissect_source_base_url}/{plugin.module.replace('.', '/')}.py", } variants.append(VARIANT_TEMPLATE.format(**info)) - title = f"``{name}``\n{(len(name) + 4) * '='}" return PAGE_TEMPLATE.format( - title=title, + title=f"``{name}``\n{(len(name) + 4) * '='}", name=name, variants="\n\n".join(variants), ) +def get_record_docs(records: list[RecordDescriptor] | RecordDescriptor) -> str: + """Generate a rst list table based on :class:`RecordDescriptor` fields.""" + + output = [] + records = records if isinstance(records, list) else [records] + for desc in records: + if not desc: + continue + + out = f""" + + {desc.name} + {"~"*len(desc.name)} + + .. list-table:: + :widths: 20 20 60 + :header-rows: 1 + + * - Field name + - Field type + - Description\n""" + + for field in desc.fields.values(): + out += f" * - ``{field.name}``\n - ``{field.typename}``\n - \n" + + output.append(textwrap.dedent(out)) + + return "\n\n".join(["This plugin can output the following records.", *output] if output else [*output]) + + def setup(app: Sphinx) -> dict[str, Any]: app.connect("builder-inited", builder_inited) app.connect("build-finished", build_finished) app.add_config_value("dissect_plugins_keep_files", False, "html") + app.add_config_value("dissect_source_base_url", "https://github.com/fox-it/dissect.target/tree/main", "html") return { "version": "0.1", diff --git a/docs/source/plugins/index.rst b/docs/source/plugins/index.rst index 44b41f0..7fd776c 100644 --- a/docs/source/plugins/index.rst +++ b/docs/source/plugins/index.rst @@ -5,4 +5,5 @@ Plugin Reference :maxdepth: 1 :glob: + /plugins/*/index /plugins/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..977d8ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,88 @@ +[build-system] +requires = ["setuptools>=80.9.0", "setuptools_scm[toml]>=6.4.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dissect-docs" +description = "" +readme = "README.md" +requires-python = ">=3.10.0" +license = "AGPL-3.0-or-later" +license-files = ["LICENSE", "COPYRIGHT"] +dynamic = ["version"] + +dependencies = [ + "dissect", + "sphinx", + "sphinx-autoapi", + "sphinx_argparse_cli", + "sphinx-copybutton", + "sphinx-design", + "sphinx-autobuild", + "furo", + # Linting + "ruff==0.12.9", + # Needed to generate docs for dissect.target and flow.record. + "defusedxml", + "msgpack", + "structlog", + # Needed to generate docs for dissect.target and dissect.fve. + "pycryptodome", + "argon2-cffi", +] + +[tool.ruff] +line-length = 120 +required-version = ">=0.12.0" + +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +select = [ + "F", + "E", + "W", + "I", + "UP", + "YTT", + "ANN", + "B", + "C4", + "DTZ", + "T10", + "FA", + "ISC", + "G", + "INP", + "PIE", + "PYI", + "PT", + "Q", + "RSE", + "RET", + "SLOT", + "SIM", + "TID", + "TCH", + "PTH", + "PLC", + "TRY", + "FLY", + "PERF", + "FURB", + "RUF", +] +ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003", "PLC0415"] + +[tool.ruff.lint.per-file-ignores] +"tests/_docs/**" = ["INP001"] + +[tool.ruff.lint.isort] +known-first-party = ["dissect-docs"] +known-third-party = ["dissect"] + +[tool.setuptools.packages.find] +include = ["dissect.*"] + +[tool.setuptools_scm] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1b6b6fb..0000000 --- a/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -sphinx -sphinx-autoapi -sphinx_argparse_cli -sphinx-copybutton -sphinx-design -sphinx-autobuild -furo - -# These dependencies are needed to generate docs for dissect.target and -# flow.record. -defusedxml -msgpack -structlog - -# These dependencies are needed to generate docs for dissect.target and -# dissect.fve. -pycryptodome==3.22.0 -argon2-cffi diff --git a/tox.ini b/tox.ini index 7af2a64..8666cca 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = lint, py3, pypy3 # requires if they are not available on the host system. This requires the # locally installed tox to have a minimum version 3.3.0. This means the names # of the configuration options are still according to the tox 3.x syntax. -minversion = 4.2.4 +minversion = 4.27.0 # This version of virtualenv will install setuptools version 65.5.0 and pip # 22.3. These versions fully support python projects defined only through a # pyproject.toml file (PEP-517/PEP-518/PEP-621) @@ -12,9 +12,6 @@ requires = virtualenv>=20.16.6 [testenv:docs-build] allowlist_externals = make -deps = - dissect - -r{toxinidir}/requirements.txt commands = make -C docs clean make -C docs html NO_AUTOAPI=1 O="--fail-on-warning"