Add supported-table directive that validates whether all different targets we support are documented (#116)

This adds a .. dissect-supported-table directive which verifies whether our tables in supported-targets.rst are still up to date

The directive itself provides the following options:

- `source-path`: where it needs to start looking for modules within the directory specified by the `dissect_projects_path` (required)
- `blacklist`: a comma separated list that tells the plugin which modules to ignore
- `glob-pattern`: a pattern to search for inside `source-path` if not specified it uses *.py by default

Author: Miauwkeru

docs/source/_ext/dissect_supported_table.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+from typing_extensions import override
+
+from sphinx.application import Sphinx
+from sphinx.util.logging import getLogger
+from sphinx.util.console import colorize
+from sphinx.addnodes import pending_xref
+from docutils.parsers.rst.directives.tables import ListTable
+from docutils.parsers.rst import directives
+from docutils import nodes
+
+
+LOGGER = getLogger(__name__)
+DISSECT_PREFIX = colorize("bold", "[Dissect] ")
+
+
+class SupportedTargetTable(ListTable):
+    """Extended list-table directive that validates if we have all our supported target modules documented."""
+
+    option_spec = ListTable.option_spec.copy()
+    option_spec["source-path"] = directives.unchanged_required
+    option_spec["blacklist"] = directives.unchanged
+    option_spec["glob-pattern"] = directives.unchanged
+
+    @override
+    def run(self):
+        # Call the parent directive to get the table node
+        result = super().run()
+
+        module_references = []
+        table_name = ""
+        # Perform custom checks on the table
+        if result and isinstance(result[0], nodes.table):
+            table_node = result[0]
+            table_name = table_node[0].astext()
+            LOGGER.info(
+                DISSECT_PREFIX
+                + colorize("darkgreen", "Gathering references from table '%s'"),
+                table_name,
+            )
+            module_references = self._gather_table_references(table_node)
+
+        if module_references:
+            self.validate_references(table_name, module_references)
+
+        return result
+
+    def _gather_table_references(self, table_node: nodes.table) -> list[str]:
+        """Gather all the references inside the table."""
+        references = []
+        for ref in table_node.findall(pending_xref):
+            target = ref.get("reftarget")
+            split_names = target.rsplit(".", 2)
+            if target.endswith("._os"):
+                modname = split_names[-2]
+            else:
+                modname = split_names[-1]
+            references.append(modname)
+
+        return references
+
+    def validate_references(self, table_name: str, module_references: list[str]):
+        LOGGER.info(
+            DISSECT_PREFIX
+            + colorize("darkgreen", "Validating module references from table '%s'"),
+            table_name,
+        )
+        check_path: str = self.options.get("source-path")
+        # Get the environment for the sphinx app
+        environment = self.state.document.settings.env
+        dissect_projects_dir = environment.config.dissect_projects_path
+
+        search_path = dissect_projects_dir / check_path
+
+        black_list = set()
+        black_list.update(["__init__"])
+        black_list.update(
+            module for module in self.options.get("blacklist", "").split(",") if module
+        )
+
+        glob = self.options.get("glob-pattern", "*.py")
+        for file in search_path.glob(glob):
+            if file.name == "_os.py":
+                file = file.parent
+
+            relative_file = file.relative_to(dissect_projects_dir)
+
+            if file.stem in black_list:
+                LOGGER.debug(
+                    DISSECT_PREFIX + colorize("darkgrey", "Skipping %s"), relative_file
+                )
+                continue
+
+            if file.stem not in module_references:
+                LOGGER.warning(
+                    DISSECT_PREFIX
+                    + colorize(
+                        "darkred", "Missing documentation entry for %s in table '%s'"
+                    ),
+                    relative_file,
+                    table_name,
+                )
+
+        LOGGER.info(
+            DISSECT_PREFIX + colorize("darkgreen", "Done validating table '%s'"),
+            table_name,
+        )
+
+
+def setup(app: Sphinx) -> dict[str, Any]:
+    app.add_config_value(
+        "dissect_projects_path", Path(__file__).parent.parent.parent.parent, "html"
+    )
+    app.add_directive("dissect-supported-table", SupportedTargetTable)
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

docs/source/conf.py

@@ -51,6 +51,7 @@
     "sphinx_copybutton",
     "sphinx_design",
     "dissect_plugins",
+    "dissect_supported_table",
 ]
 
 # Define the canonical URL if you are using a custom domain on Read the Docs
@@ -182,6 +183,8 @@
     "ref.python",
 ]
 
+dissect_projects_path = Path(__file__).parent.parent.parent / "submodules"
+
 
 def autoapi_skip_hook(app: Sphinx, what: str, name: str, obj, skip: bool, options: list[str]) -> bool:
     # Do not skip OS modules in dissect.target (caught by `private-members`)

docs/source/supported-targets.rst

@@ -32,9 +32,11 @@ If needed, you can choose the loader yourself by using ``-L <loader type>`` opti
   :ref:`supported filesystems <supported-targets:Filesystems>`. Whether a target is supported
   as a loader, a container or a filesystem depends on implementation details for that specific format.
 
-.. list-table:: Supported loaders
+.. dissect-supported-table:: Supported loaders
   :header-rows: 1
   :widths: 20 15 5
+  :source-path: dissect.target/dissect/target/loaders
+  :blacklist: cyber,phobos,itunes,target,vb,asdf,log,vmsupport,res,direct,profile,pvs
 
   * - Description
     - Format
@@ -168,9 +170,11 @@ These can be virtual machine files, forensic containers or a hard disk itself.
 Dissect can select the appropriate container automatically based on either the file extension or file magic.
 For example, the QCOW2 container gets selected if the file extension is ``.qcow2`` or if the first bytes of the file are ``b"QFI\xfb"``.
 
-.. list-table:: Supported containers
+.. dissect-supported-table:: Supported containers
   :header-rows: 1
   :widths: 15 5 5
+  :source-path: dissect.target/dissect/target/containers
+  :blacklist: raw,asdf,hds,split
 
   * - Description
     - Format
@@ -211,9 +215,10 @@ Partition Schemes and Volume Systems
 
 Dissect supports most common partition schemes. Nested partitions are supported as well.
 
-.. list-table:: Supported Partition Schemes
+.. dissect-supported-table:: Supported Partition Schemes
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.volume/dissect/volume/disk/schemes
 
   * - Description
     - API
@@ -232,9 +237,11 @@ Besides these standard partition schemes, Dissect supports disks in RAID configu
 
   For more details, see :doc:`volumes <advanced/volumes>`.
 
-.. list-table:: Supported volume systems
+.. dissect-supported-table:: Supported volume systems
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.target/dissect/target/volumes
+  :blacklist: disk,luks,bde
 
   * - Description
     - API
@@ -250,9 +257,11 @@ Besides these standard partition schemes, Dissect supports disks in RAID configu
 Dissect also has decryption capability for some well known systems.
 This functionality can be accessed with a keychain file (specified with ``-K``) with multiple passphrases or a keychain value (``-Kv``) in most Dissect tools.
 
-.. list-table:: Supported encrypted volume systems
+.. dissect-supported-table:: Supported encrypted volume systems
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.target/dissect/target/volumes
+  :blacklist: disk,ddf,lvm,md,vmfs
 
   * - Description
     - API
@@ -275,9 +284,11 @@ or need implementation in different areas to work correctly.
 
   For more details, see :doc:`Filesystems </advanced/filesystems>`.
 
-.. list-table:: Supported filesystems
+.. dissect-supported-table:: Supported filesystems
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.target/dissect/target/filesystems
+  :blacklist: zip,smb,itunes,cb,overlay,ntds,tar,dir
 
   * - Description
     - API
@@ -324,9 +335,12 @@ Operating Systems
 Dissect tries to automatically figure out what operating system is available on the target, based on known file locations and structures.
 Once the operating system is known, it enables you to get more accurate information from the system, for example, the user or network configuration.
 
-.. list-table:: Supported operating systems
+.. dissect-supported-table:: Supported operating systems
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.target/dissect/target/plugins
+  :glob-pattern: **/_os.py
+  :blacklist: default
 
   * - Description
     - API
@@ -379,9 +393,10 @@ It can do this recursively, and look for *child targets* inside the *child targe
 
   For more details, see :ref:`Child targets <advanced/targets:Targets in targets>`.
 
-.. list-table:: Supported child targets
+.. dissect-supported-table:: Supported child targets
   :header-rows: 1
   :widths: 20 5
+  :source-path: dissect.target/dissect/target/plugins/child
 
   * - Description
     - API