Skip to content

Add record descriptor info to plugin doc pages #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JSCU-CNI wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add record descriptor info to plugin doc pages #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ef9dd9
add record descriptor info to plugin doc pages
JSCU-CNI Sep 11, 2025
0c8473a
Merge branch 'main' into record-descriptor-docs
JSCU-CNI Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
coverage.xml
.coverage
dist/
.eggs/
*.egg-info/
*.pyc
__pycache__/
.pytest_cache/
tests/_docs/api
tests/_docs/build
.tox/

build/
_build/
__pycache__

docs/source/api/*/
1 change: 1 addition & 0 deletions docs/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ help:

clean: Makefile
rm -rf source/api/*/
rm -rf source/plugins/*
@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

watch: Makefile
Expand Down
105 changes: 90 additions & 15 deletions docs/source/_ext/dissect_plugins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
from __future__ import annotations

import textwrap
from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING, Any

from dissect.target.exceptions import PluginError
from dissect.target.helpers import docs
Expand All @@ -9,6 +12,9 @@
from sphinx.util.display import status_iterator
from sphinx.util.logging import getLogger

if TYPE_CHECKING:
from flow.record import RecordDescriptor

LOGGER = getLogger(__name__)

PAGE_TEMPLATE = """..
Expand All @@ -24,30 +30,54 @@
"""

VARIANT_TEMPLATE = """
.. list-table:: Details
.. list-table::
:widths: 20 80

* - Module
- ``{module}.{class}``
* - Output
- ``{output}``
* - Source
- {source}

**Module documentation**
Module documentation
--------------------

{class_doc}

**Function documentation**
Function documentation
----------------------

{func_doc}

Record output
-------------

{record_doc}
"""

NAMESPACE_TEMPLATE = """
This is a namespace plugin. This means that by running this plugin, it will automatically run
all other plugins under this namespace:

.. toctree::
:maxdepth: 1
:glob:

{exports}
"""

INDEX_TEMPLATE = """\
Plugin Reference
================

.. toctree::
:maxdepth: 1
:glob:

/plugins/*/index
/plugins/*
"""

def builder_inited(app: Sphinx) -> None:
dst = Path(app.srcdir).joinpath("plugins")
Expand All @@ -58,9 +88,13 @@ def builder_inited(app: Sphinx) -> None:

for plugin in plugins():
# Ignore all modules in general as those are all internal or utility
if plugin.module.startswith("general."):
if plugin.path.startswith("general."):
continue

# # Exclude plugins without any exports and/or InternalPlugins
# if not plugin.findable or not plugin.exports:
# continue

if ns := plugin.namespace:
plugin_map.setdefault(ns, []).append(plugin)

Expand All @@ -72,17 +106,24 @@ def builder_inited(app: Sphinx) -> None:
export = f"{ns}.{export}"
plugin_map.setdefault(export, []).append(plugin)

dst.joinpath("index.rst").write_text(INDEX_TEMPLATE)

for name, plugin in status_iterator(
plugin_map.items(),
colorize("bold", "[Dissect] Writing plugin files... "),
length=len(plugin_map),
stringify_func=(lambda x: x[0]),
):
dst_path = dst.joinpath(name + ".rst")
if dst_path.exists():
continue

dst_path.write_text(_format_template(name, plugin))
if name == plugin[0].namespace:
dst_path = dst.joinpath(f"{name.replace('.', '/')}/index.rst")
else:
dst_path = dst.joinpath(name.replace(".", "/") + ".rst")

if not dst_path.parent.is_dir():
dst_path.parent.mkdir()

dst_path.write_text(_format_template(app, name, plugin))


def build_finished(app: Sphinx, exception: Exception) -> None:
Expand All @@ -91,14 +132,14 @@ def build_finished(app: Sphinx, exception: Exception) -> None:
if app.verbosity > 1:
LOGGER.info(colorize("bold", "[Dissect] ") + colorize("darkgreen", "Cleaning generated .rst files"))

for rst in dst.glob("*.rst"):
with open(rst, "rb") as fh:
for rst in dst.glob("**.rst"):
with rst.open("rb") as fh:
if fh.read(16) != b"..\n generated":
continue
rst.unlink()


def _format_template(name: str, plugins: list[dict]) -> str:
def _format_template(app: Sphinx, name: str, plugins: list[dict]) -> str:
func_name = name.split(".")[-1] if "." in name else name
variants = []

Expand All @@ -117,35 +158,69 @@ def _format_template(name: str, plugins: list[dict]) -> str:
func_output = "records"
func_doc = NAMESPACE_TEMPLATE.format(
exports="\n".join(
f"- :doc:`/plugins/{ns}.{export}`" for export in plugin.exports if export != "__call__"
f" /plugins/{ns.replace('.', '/')}/{export}.rst" for export in plugin.exports if export != "__call__"
)
)
record_doc = get_record_docs(plugin.cls.__record_descriptors__) if hasattr(plugin.cls, "__record_descriptors__") else ""
else:
func = getattr(plugin_class, func_name)
func_output, func_doc = docs._get_func_details(func)
record_doc = get_record_docs(func.__record__) if hasattr(func, "__record__") else "This plugin does not generate record output."

info = {
"module": plugin.module,
"class": plugin.qualname,
"output": func_output,
"class_doc": class_doc,
"func_doc": func_doc,
"record_doc": record_doc,
"source": f"{app.config.dissect_source_base_url}/{plugin.module.replace('.', '/')}.py",
}

variants.append(VARIANT_TEMPLATE.format(**info))

title = f"``{name}``\n{(len(name) + 4) * '='}"
return PAGE_TEMPLATE.format(
title=title,
title=f"``{name}``\n{(len(name) + 4) * '='}",
name=name,
variants="\n\n".join(variants),
)


def get_record_docs(records: list[RecordDescriptor] | RecordDescriptor) -> str:
"""Generate a rst list table based on :class:`RecordDescriptor` fields."""

output = []
records = records if isinstance(records, list) else [records]
for desc in records:
if not desc:
continue

out = f"""

{desc.name}
{"~"*len(desc.name)}

.. list-table::
:widths: 20 20 60
:header-rows: 1

* - Field name
- Field type
- Description\n"""

for field in desc.fields.values():
out += f" * - ``{field.name}``\n - ``{field.typename}``\n - \n"

output.append(textwrap.dedent(out))

return "\n\n".join(["This plugin can output the following records.", *output] if output else [*output])


def setup(app: Sphinx) -> dict[str, Any]:
app.connect("builder-inited", builder_inited)
app.connect("build-finished", build_finished)
app.add_config_value("dissect_plugins_keep_files", False, "html")
app.add_config_value("dissect_source_base_url", "https://github.com/fox-it/dissect.target/tree/main", "html")

return {
"version": "0.1",
Expand Down
1 change: 1 addition & 0 deletions docs/source/plugins/index.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ Plugin Reference
:maxdepth: 1
:glob:

/plugins/*/index
/plugins/*
88 changes: 88 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
[build-system]
requires = ["setuptools>=80.9.0", "setuptools_scm[toml]>=6.4.0"]
build-backend = "setuptools.build_meta"

[project]
name = "dissect-docs"
description = ""
readme = "README.md"
requires-python = ">=3.10.0"
license = "AGPL-3.0-or-later"
license-files = ["LICENSE", "COPYRIGHT"]
dynamic = ["version"]

dependencies = [
"dissect",
"sphinx",
"sphinx-autoapi",
"sphinx_argparse_cli",
"sphinx-copybutton",
"sphinx-design",
"sphinx-autobuild",
"furo",
# Linting
"ruff==0.12.9",
# Needed to generate docs for dissect.target and flow.record.
"defusedxml",
"msgpack",
"structlog",
# Needed to generate docs for dissect.target and dissect.fve.
"pycryptodome",
"argon2-cffi",
]

[tool.ruff]
line-length = 120
required-version = ">=0.12.0"

[tool.ruff.format]
docstring-code-format = true

[tool.ruff.lint]
select = [
"F",
"E",
"W",
"I",
"UP",
"YTT",
"ANN",
"B",
"C4",
"DTZ",
"T10",
"FA",
"ISC",
"G",
"INP",
"PIE",
"PYI",
"PT",
"Q",
"RSE",
"RET",
"SLOT",
"SIM",
"TID",
"TCH",
"PTH",
"PLC",
"TRY",
"FLY",
"PERF",
"FURB",
"RUF",
]
ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003", "PLC0415"]

[tool.ruff.lint.per-file-ignores]
"tests/_docs/**" = ["INP001"]

[tool.ruff.lint.isort]
known-first-party = ["dissect-docs"]
known-third-party = ["dissect"]

[tool.setuptools.packages.find]
include = ["dissect.*"]

[tool.setuptools_scm]
18 changes: 0 additions & 18 deletions requirements.txt
View file
Open in desktop

This file was deleted.

5 changes: 1 addition & 4 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,14 @@ envlist = lint, py3, pypy3
# requires if they are not available on the host system. This requires the
# locally installed tox to have a minimum version 3.3.0. This means the names
# of the configuration options are still according to the tox 3.x syntax.
minversion = 4.2.4
minversion = 4.27.0
# This version of virtualenv will install setuptools version 65.5.0 and pip
# 22.3. These versions fully support python projects defined only through a
# pyproject.toml file (PEP-517/PEP-518/PEP-621)
requires = virtualenv>=20.16.6

[testenv:docs-build]
allowlist_externals = make
deps =
dissect
-r{toxinidir}/requirements.txt
commands =
make -C docs clean
make -C docs html NO_AUTOAPI=1 O="--fail-on-warning"