Add NTDS parser

.github/workflows/dissect-ci.yml

@@ -12,6 +12,8 @@ jobs:
   ci:
     uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-template.yml@main
     secrets: inherit
+    with:
+      run-benchmarks: true
 
   publish:
     if: ${{ github.ref_name == 'main' || github.ref_type == 'tag' }}

MANIFEST.in

@@ -1,2 +1,4 @@
+exclude .gitattributes
 exclude .gitignore
 recursive-exclude .github/ *
+recursive-exclude tests/_data/ *

dissect/database/ese/btree.py

@@ -116,7 +116,8 @@ def search(self, key: bytes, exact: bool = True) -> Node:
         """
         page = self._page
         while True:
-            node = find_node(page, key)
+            num = find_node(page, key)
+            node = page.node(num)
 
             if page.is_branch:
                 page = self.db.page(node.child)
@@ -132,35 +133,45 @@ def search(self, key: bytes, exact: bool = True) -> Node:
         return self.node()
 
 
-def find_node(page: Page, key: bytes) -> Node:
+def find_node(page: Page, key: bytes) -> int:
     """Search a page for a node matching ``key``.
 
+    Referencing Extensible-Storage-Engine source, they bail out early if they find an exact match.
+    However, we prefer to always find the _first_ node that is greater than or equal to the key,
+    so we can handle cases where there are duplicate index keys. This is important for "range" searches
+    where we want to find all keys matching a certain prefix, and not end up somewhere in the middle of the range.
+
     Args:
         page: The page to search.
         key: The key to search.
+
+    Returns:
+        The node number of the first node that's greater than or equal to the key.
     """
-    first_node_idx = 0
-    last_node_idx = page.node_count - 1
+    lo, hi = 0, page.node_count - 1
+    res = 0
 
     node = None
-    while first_node_idx < last_node_idx:
-        node_idx = (first_node_idx + last_node_idx) // 2
-        node = page.node(node_idx)
+    while lo < hi:
+        mid = (lo + hi) // 2
+        node = page.node(mid)
 
         # It turns out that the way BTree keys are compared matches 1:1 with how Python compares bytes
         # First compare data, then length
-        if key < node.key:
-            last_node_idx = node_idx
-        elif key == node.key:
-            if page.is_branch:
-                # If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
-                # Page keys for branch pages appear to be non-inclusive upper bounds
-                node_idx = min(node_idx + 1, page.node_count - 1)
-                node = page.node(node_idx)
+        res = (key < node.key) - (key > node.key)
 
-            return node
+        if res < 0:
+            lo = mid + 1
         else:
-            first_node_idx = node_idx + 1
+            hi = mid
+
+    # Final comparison on the last node
+    node = page.node(lo)
+    res = (key < node.key) - (key > node.key)
+
+    if page.is_branch and res == 0:
+        # If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
+        # Page keys for branch pages appear to be non-inclusive upper bounds
+        lo = min(lo + 1, page.node_count - 1)
 
-    # We're at the last node
-    return page.node(first_node_idx)
+    return lo

dissect/database/ese/c_ese.py

@@ -231,7 +231,7 @@
     BKINFOTYPE  bkinfoTypeCopyPrev;                 //  Type of Last successful Incremental backup
     BKINFOTYPE  bkinfoTypeDiffPrev;                 //  Type of Last successful Differential backup
 
-// 476 bytes
+//  476 bytes
     ULONG       ulIncrementalReseedCount;           //  number of times incremental reseed has been initiated on this database
     LOGTIME     logtimeIncrementalReseed;           //  the date of the last time that incremental reseed was initiated on this database
     ULONG       ulIncrementalReseedCountOld;        //  number of times incremental reseed was initiated on this database before the last defrag
@@ -240,7 +240,7 @@
     LOGTIME     logtimePagePatch;                   //  the date of the last time that a page was patched as a part of incremental reseed
     ULONG       ulPagePatchCountOld;                //  number of pages patched in the database as a part of incremental reseed before the last defrag
 
-// 508 bytes
+//  508 bytes
     QWORD       qwSortVersion;                      // DEPRECATED: In old versions had "default" (?English?) LCID version, in new versions has 0xFFFFFFFFFFFF.
 
 //  516 bytes                                       // checksum during recovery state
@@ -399,6 +399,22 @@
     Encrypted               = 0x40,                 // fEncrypted
 };
 
+flag FIELDFLAG : uint16 {
+    NotNull                 = 0x0001,               // NULL values not allowed
+    Version                 = 0x0002,               // Version field
+    Autoincrement           = 0x0004,               // Autoincrement field
+    Multivalued             = 0x0008,               // Multi-valued column
+    Default                 = 0x0010,               // Column has ISAM default value
+    EscrowUpdate            = 0x0020,               // Escrow updated column
+    Finalize                = 0x0040,               // Finalizable column
+    UserDefinedDefault      = 0x0080,               // The default value is generated through a callback
+    TemplateColumnESE98     = 0x0100,               // Template table column created in ESE98 (ie. fDerived bit will be set in TAGFLD of records of derived tables)
+    DeleteOnZero            = 0x0200,               // DeleteOnZero column
+    PrimaryIndexPlaceholder = 0x0800,               // Field is no longer in primary index, but must be retained as a placeholder
+    Compressed              = 0x1000,               // Data stored in the column should be compressed
+    Encrypted               = 0x2000,               // Data stored in the column is encrypted
+};
+
 flag JET_bitIndex : uint32 {
     Unique                  = 0x00000001,
     Primary                 = 0x00000002,
@@ -467,5 +483,6 @@
 TAGFLD_HEADER = c_ese.TAGFLD_HEADER
 CODEPAGE = c_ese.CODEPAGE
 COMPRESSION_SCHEME = c_ese.COMPRESSION_SCHEME
+FIELDFLAG = c_ese.FIELDFLAG
 IDBFLAG = c_ese.IDBFLAG
 IDXFLAG = c_ese.IDXFLAG

dissect/database/ese/c_ese.pyi

@@ -426,6 +426,21 @@ class _c_ese(__cs__.cstruct):
         Null = ...
         Encrypted = ...
 
+    class FIELDFLAG(__cs__.Flag):
+        NotNull = ...
+        Version = ...
+        Autoincrement = ...
+        Multivalued = ...
+        Default = ...
+        EscrowUpdate = ...
+        Finalize = ...
+        UserDefinedDefault = ...
+        TemplateColumnESE98 = ...
+        DeleteOnZero = ...
+        PrimaryIndexPlaceholder = ...
+        Compressed = ...
+        Encrypted = ...
+
     class JET_bitIndex(__cs__.Flag):
         Unique = ...
         Primary = ...
@@ -487,5 +502,6 @@ TAG_FLAG: TypeAlias = c_ese.TAG_FLAG
 TAGFLD_HEADER: TypeAlias = c_ese.TAGFLD_HEADER
 CODEPAGE: TypeAlias = c_ese.CODEPAGE
 COMPRESSION_SCHEME: TypeAlias = c_ese.COMPRESSION_SCHEME
+FIELDFLAG: TypeAlias = c_ese.FIELDFLAG
 IDBFLAG: TypeAlias = c_ese.IDBFLAG
 IDXFLAG: TypeAlias = c_ese.IDXFLAG

dissect/database/ese/cursor.py

@@ -3,12 +3,14 @@
 from typing import TYPE_CHECKING
 
 from dissect.database.ese.btree import BTree
-from dissect.database.ese.exception import NoNeighbourPageError
+from dissect.database.ese.exception import KeyNotFoundError, NoNeighbourPageError
 from dissect.database.ese.record import Record
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
+    from typing_extensions import Self
+
     from dissect.database.ese.index import Index
     from dissect.database.ese.page import Node
     from dissect.database.ese.util import RecordValue
@@ -30,13 +32,10 @@ def __init__(self, index: Index):
         self._secondary = None if index.is_primary else BTree(self.db, self.table.root)
 
     def __iter__(self) -> Iterator[Record]:
-        while True:
-            yield self._record()
-
-            try:
-                self._primary.next()
-            except NoNeighbourPageError:
-                break
+        record = self.record()
+        while record is not None:
+            yield record
+            record = self.next()
 
     def _node(self) -> Node:
         """Return the node the cursor is currently on. Resolves the secondary index if needed.
@@ -50,33 +49,84 @@ def _node(self) -> Node:
             node = self._secondary.search(node.data.tobytes(), exact=True)
         return node
 
-    def _record(self) -> Record:
+    def record(self) -> Record:
         """Return the record the cursor is currently on.
 
         Returns:
             A :class:`~dissect.database.ese.record.Record` object of the current record.
         """
         return Record(self.table, self._node())
 
-    def reset(self) -> None:
+    def reset(self) -> Self:
         """Reset the internal state."""
         self._primary.reset()
         if self._secondary:
             self._secondary.reset()
+        return self
+
+    def next(self) -> Record | None:
+        """Move the cursor to the next record and return it.
+
+        Can move the cursor to the next page as a side effect.
 
-    def search(self, **kwargs: RecordValue) -> Record:
+        Returns:
+            A :class:`~dissect.database.ese.record.Record` object of the next record.
+        """
+        try:
+            self._primary.next()
+        except NoNeighbourPageError:
+            return None
+        return self.record()
+
+    def prev(self) -> Record | None:
+        """Move the cursor to the previous node and return it.
+
+        Can move the cursor to the previous page as a side effect.
+
+        Returns:
+            A :class:`~dissect.database.ese.record.Record` object of the previous record.
+        """
+        try:
+            self._primary.prev()
+        except NoNeighbourPageError:
+            return None
+        return self.record()
+
+    def make_key(self, *args: RecordValue, **kwargs: RecordValue) -> bytes:
+        """Generate a key for this index from the given values.
+
+        Args:
+            *args: The values to generate a key for.
+            **kwargs: The columns and values to generate a key for.
+
+        Returns:
+            The generated key as bytes.
+        """
+        if not args and not kwargs:
+            raise ValueError("At least one value must be provided")
+
+        if args and kwargs:
+            raise ValueError("Cannot mix positional and keyword arguments in make_key")
+
+        if args and not len(args) == 1 and not isinstance(args[0], list):
+            raise ValueError("When using positional arguments, provide a single list of values")
+
+        return self.index.make_key(args[0] if args else kwargs)
+
+    def search(self, *args: RecordValue, **kwargs: RecordValue) -> Record:
         """Search the index for the requested values.
 
         Searching modifies the cursor state. Searching again will search from the current position.
         Reset the cursor with :meth:`reset` to start from the beginning.
 
         Args:
+            *args: The values to search for.
             **kwargs: The columns and values to search for.
 
         Returns:
             A :class:`~dissect.database.ese.record.Record` object of the found record.
         """
-        key = self.index.make_key(kwargs)
+        key = self.make_key(*args, **kwargs)
         return self.search_key(key, exact=True)
 
     def search_key(self, key: bytes, exact: bool = True) -> Record:
@@ -88,24 +138,27 @@ def search_key(self, key: bytes, exact: bool = True) -> Record:
                    next record that is greater than or equal to the key.
         """
         self._primary.search(key, exact)
-        return self._record()
+        return self.record()
 
-    def seek(self, **kwargs: RecordValue) -> None:
+    def seek(self, *args: RecordValue, **kwargs: RecordValue) -> Self:
         """Seek to the record with the given values.
 
         Args:
+            *args: The values to seek to.
             **kwargs: The columns and values to seek to.
         """
-        key = self.index.make_key(kwargs)
+        key = self.make_key(*args, **kwargs)
         self.search_key(key, exact=False)
+        return self
 
-    def seek_key(self, key: bytes) -> None:
+    def seek_key(self, key: bytes) -> Self:
         """Seek to the record with the given ``key``.
 
         Args:
             key: The key to seek to.
         """
         self._primary.search(key, exact=False)
+        return self
 
     def find(self, **kwargs: RecordValue) -> Record | None:
         """Find a record in the index.
@@ -130,7 +183,10 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
         other_columns = kwargs
 
         # We need at least an exact match on the indexed columns
-        self.search(**indexed_columns)
+        try:
+            self.search(**indexed_columns)
+        except KeyNotFoundError:
+            return
 
         current_key = self._primary.node().key
 
@@ -150,7 +206,7 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
             if current_key != self._primary.node().key:
                 break
 
-            record = self._record()
+            record = self.record()
             for k, v in other_columns.items():
                 value = record.get(k)
                 # If the record value is a list, we do a check based on the queried value
@@ -172,39 +228,3 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
                 self._primary.next()
             except NoNeighbourPageError:
                 break
-
-    def record(self) -> Record:
-        """Return the record the cursor is currently on.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the current record.
-        """
-        return self._record()
-
-    def next(self) -> Record:
-        """Move the cursor to the next record and return it.
-
-        Can move the cursor to the next page as a side effect.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the next record.
-        """
-        try:
-            self._primary.next()
-        except NoNeighbourPageError:
-            raise IndexError("No next record")
-        return self._record()
-
-    def prev(self) -> Record:
-        """Move the cursor to the previous node and return it.
-
-        Can move the cursor to the previous page as a side effect.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the previous record.
-        """
-        try:
-            self._primary.prev()
-        except NoNeighbourPageError:
-            raise IndexError("No previous record")
-        return self._record()

dissect/database/ese/ese.py

@@ -36,6 +36,7 @@ def __init__(self, fh: BinaryIO, impacket_compat: bool = False):
         self.fh = fh
         self.impacket_compat = impacket_compat
 
+        self.fh.seek(0)
         self.header = c_ese.DBFILEHDR(fh)
         if self.header.ulMagic != ulDAEMagic:
             raise InvalidDatabase("invalid file header signature")