Skip to content

Add NTDS parser #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Schamper merged 42 commits into from
Jan 22, 2026
Merged

Add NTDS parser #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
b01ac8e
Added ntds.dit and SYSTEM hive test data
kylebullock820 Oct 24, 2025
6976f9c
Add basic NTDS parser + tests
kylebullock820 Oct 31, 2025
fea2af4
Renaming and added proper docstrings
kylebullock820 Nov 5, 2025
e099bba
Add benchmark tests along with a larger test file
kylebullock820 Nov 5, 2025
b0b8030
Initial refactor
Schamper Dec 11, 2025
8410926
Refactor security descriptors
Schamper Dec 11, 2025
6773c65
More cleanup
Schamper Dec 11, 2025
b1cdf19
More changes
Schamper Dec 17, 2025
ae34ce4
Tweak cursor
Schamper Dec 17, 2025
2dd01e8
More tweaks
Schamper Dec 18, 2025
659e1a2
Fix multi values
Schamper Dec 18, 2025
d6b6152
More references
Schamper Dec 18, 2025
0c9c12a
More changes
Schamper Dec 18, 2025
d4736a0
This shit is like crack
Schamper Dec 19, 2025
f9fb7b3
Add GOAD ntds.dit
Schamper Dec 19, 2025
61f19e1
Small tweaks, add trusts
Schamper Dec 19, 2025
0456f1e
Add funny memes
Schamper Dec 19, 2025
e4f3f6b
Snorting lines of this
Schamper Dec 19, 2025
78c5121
Straight into my veins 💉
Schamper Dec 19, 2025
06f6672
Add a link for a future lucky soul
Schamper Dec 19, 2025
1070992
Fix linter
Schamper Dec 19, 2025
d8eb9ce
WIP update tests
Schamper Dec 30, 2025
69d5eb5
Fix unit tests
Schamper Jan 2, 2026
3301854
Add PEK support
Schamper Jan 2, 2026
b486273
Improve PEK and DN
Schamper Jan 5, 2026
0ad313e
Fix for phantom objects
Schamper Jan 5, 2026
5e4092a
Add docstrings
Schamper Jan 5, 2026
d6b94b4
Cleanup object repr
Schamper Jan 5, 2026
6f98843
Fix linting and tests
Schamper Jan 6, 2026
0073d20
Add parsing of supplemental credentials
Schamper Jan 6, 2026
5bb48bb
Add rid property
Schamper Jan 6, 2026
2028943
Take this away from me
Schamper Jan 7, 2026
312ad48
Remove small ntds.dit
Schamper Jan 9, 2026
cb11120
Simplify benchmarks
Schamper Jan 9, 2026
4d46d29
Prevent caching influencing benchmarks
Schamper Jan 9, 2026
08fcd40
Run benchmarks
Schamper Jan 13, 2026
11b0c1e
Remove SYSTEM hive and document syskey
Schamper Jan 13, 2026
7c65d0c
Review
Schamper Jan 20, 2026
42050be
Update return types
Schamper Jan 21, 2026
b074955
Small changes
Schamper Jan 22, 2026
7c574e5
Replace large ntds.dit file with snapshotted version
kylebullock820 Jan 22, 2026
3e0aa64
Merge branch 'main' into feature/ntds
Schamper Jan 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/dissect-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ jobs:
ci:
uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-template.yml@main
secrets: inherit
with:
run-benchmarks: true

publish:
if: ${{ github.ref_name == 'main' || github.ref_type == 'tag' }}
Expand Down
2 changes: 2 additions & 0 deletions MANIFEST.in
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
exclude .gitattributes
exclude .gitignore
recursive-exclude .github/ *
recursive-exclude tests/_data/ *
49 changes: 30 additions & 19 deletions dissect/database/ese/btree.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ def search(self, key: bytes, exact: bool = True) -> Node:
"""
page = self._page
while True:
node = find_node(page, key)
num = find_node(page, key)
node = page.node(num)

if page.is_branch:
page = self.db.page(node.child)
Expand All @@ -132,35 +133,45 @@ def search(self, key: bytes, exact: bool = True) -> Node:
return self.node()


def find_node(page: Page, key: bytes) -> Node:
def find_node(page: Page, key: bytes) -> int:
"""Search a page for a node matching ``key``.

Referencing Extensible-Storage-Engine source, they bail out early if they find an exact match.
However, we prefer to always find the _first_ node that is greater than or equal to the key,
so we can handle cases where there are duplicate index keys. This is important for "range" searches
where we want to find all keys matching a certain prefix, and not end up somewhere in the middle of the range.

Args:
page: The page to search.
key: The key to search.

Returns:
The node number of the first node that's greater than or equal to the key.
"""
first_node_idx = 0
last_node_idx = page.node_count - 1
lo, hi = 0, page.node_count - 1
res = 0

node = None
while first_node_idx < last_node_idx:
node_idx = (first_node_idx + last_node_idx) // 2
node = page.node(node_idx)
while lo < hi:
mid = (lo + hi) // 2
node = page.node(mid)

# It turns out that the way BTree keys are compared matches 1:1 with how Python compares bytes
# First compare data, then length
if key < node.key:
last_node_idx = node_idx
elif key == node.key:
if page.is_branch:
# If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
# Page keys for branch pages appear to be non-inclusive upper bounds
node_idx = min(node_idx + 1, page.node_count - 1)
node = page.node(node_idx)
res = (key < node.key) - (key > node.key)

return node
if res < 0:
lo = mid + 1
else:
first_node_idx = node_idx + 1
hi = mid

# Final comparison on the last node
node = page.node(lo)
res = (key < node.key) - (key > node.key)

if page.is_branch and res == 0:
# If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
# Page keys for branch pages appear to be non-inclusive upper bounds
lo = min(lo + 1, page.node_count - 1)

# We're at the last node
return page.node(first_node_idx)
return lo
21 changes: 19 additions & 2 deletions dissect/database/ese/c_ese.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@
BKINFOTYPE bkinfoTypeCopyPrev; // Type of Last successful Incremental backup
BKINFOTYPE bkinfoTypeDiffPrev; // Type of Last successful Differential backup

// 476 bytes
// 476 bytes
ULONG ulIncrementalReseedCount; // number of times incremental reseed has been initiated on this database
LOGTIME logtimeIncrementalReseed; // the date of the last time that incremental reseed was initiated on this database
ULONG ulIncrementalReseedCountOld; // number of times incremental reseed was initiated on this database before the last defrag
Expand All @@ -240,7 +240,7 @@
LOGTIME logtimePagePatch; // the date of the last time that a page was patched as a part of incremental reseed
ULONG ulPagePatchCountOld; // number of pages patched in the database as a part of incremental reseed before the last defrag

// 508 bytes
// 508 bytes
QWORD qwSortVersion; // DEPRECATED: In old versions had "default" (?English?) LCID version, in new versions has 0xFFFFFFFFFFFF.

// 516 bytes // checksum during recovery state
Expand Down Expand Up @@ -399,6 +399,22 @@
Encrypted = 0x40, // fEncrypted
};

flag FIELDFLAG : uint16 {
NotNull = 0x0001, // NULL values not allowed
Version = 0x0002, // Version field
Autoincrement = 0x0004, // Autoincrement field
Multivalued = 0x0008, // Multi-valued column
Default = 0x0010, // Column has ISAM default value
EscrowUpdate = 0x0020, // Escrow updated column
Finalize = 0x0040, // Finalizable column
UserDefinedDefault = 0x0080, // The default value is generated through a callback
TemplateColumnESE98 = 0x0100, // Template table column created in ESE98 (ie. fDerived bit will be set in TAGFLD of records of derived tables)
DeleteOnZero = 0x0200, // DeleteOnZero column
PrimaryIndexPlaceholder = 0x0800, // Field is no longer in primary index, but must be retained as a placeholder
Compressed = 0x1000, // Data stored in the column should be compressed
Encrypted = 0x2000, // Data stored in the column is encrypted
};

flag JET_bitIndex : uint32 {
Unique = 0x00000001,
Primary = 0x00000002,
Expand Down Expand Up @@ -467,5 +483,6 @@
TAGFLD_HEADER = c_ese.TAGFLD_HEADER
CODEPAGE = c_ese.CODEPAGE
COMPRESSION_SCHEME = c_ese.COMPRESSION_SCHEME
FIELDFLAG = c_ese.FIELDFLAG
IDBFLAG = c_ese.IDBFLAG
IDXFLAG = c_ese.IDXFLAG
16 changes: 16 additions & 0 deletions dissect/database/ese/c_ese.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,21 @@ class _c_ese(__cs__.cstruct):
Null = ...
Encrypted = ...

class FIELDFLAG(__cs__.Flag):
NotNull = ...
Version = ...
Autoincrement = ...
Multivalued = ...
Default = ...
EscrowUpdate = ...
Finalize = ...
UserDefinedDefault = ...
TemplateColumnESE98 = ...
DeleteOnZero = ...
PrimaryIndexPlaceholder = ...
Compressed = ...
Encrypted = ...

class JET_bitIndex(__cs__.Flag):
Unique = ...
Primary = ...
Expand Down Expand Up @@ -487,5 +502,6 @@ TAG_FLAG: TypeAlias = c_ese.TAG_FLAG
TAGFLD_HEADER: TypeAlias = c_ese.TAGFLD_HEADER
CODEPAGE: TypeAlias = c_ese.CODEPAGE
COMPRESSION_SCHEME: TypeAlias = c_ese.COMPRESSION_SCHEME
FIELDFLAG: TypeAlias = c_ese.FIELDFLAG
IDBFLAG: TypeAlias = c_ese.IDBFLAG
IDXFLAG: TypeAlias = c_ese.IDXFLAG
128 changes: 74 additions & 54 deletions dissect/database/ese/cursor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
from typing import TYPE_CHECKING

from dissect.database.ese.btree import BTree
from dissect.database.ese.exception import NoNeighbourPageError
from dissect.database.ese.exception import KeyNotFoundError, NoNeighbourPageError
from dissect.database.ese.record import Record

if TYPE_CHECKING:
from collections.abc import Iterator

from typing_extensions import Self

from dissect.database.ese.index import Index
from dissect.database.ese.page import Node
from dissect.database.ese.util import RecordValue
Expand All @@ -30,13 +32,10 @@ def __init__(self, index: Index):
self._secondary = None if index.is_primary else BTree(self.db, self.table.root)

def __iter__(self) -> Iterator[Record]:
while True:
yield self._record()

try:
self._primary.next()
except NoNeighbourPageError:
break
record = self.record()
while record is not None:
yield record
record = self.next()

def _node(self) -> Node:
"""Return the node the cursor is currently on. Resolves the secondary index if needed.
Expand All @@ -50,33 +49,84 @@ def _node(self) -> Node:
node = self._secondary.search(node.data.tobytes(), exact=True)
return node

def _record(self) -> Record:
def record(self) -> Record:
"""Return the record the cursor is currently on.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the current record.
"""
return Record(self.table, self._node())

def reset(self) -> None:
def reset(self) -> Self:
"""Reset the internal state."""
self._primary.reset()
if self._secondary:
self._secondary.reset()
return self

def next(self) -> Record | None:
"""Move the cursor to the next record and return it.

Can move the cursor to the next page as a side effect.

def search(self, **kwargs: RecordValue) -> Record:
Returns:
A :class:`~dissect.database.ese.record.Record` object of the next record.
"""
try:
self._primary.next()
except NoNeighbourPageError:
return None
return self.record()

def prev(self) -> Record | None:
"""Move the cursor to the previous node and return it.

Can move the cursor to the previous page as a side effect.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the previous record.
"""
try:
self._primary.prev()
except NoNeighbourPageError:
return None
return self.record()

def make_key(self, *args: RecordValue, **kwargs: RecordValue) -> bytes:
"""Generate a key for this index from the given values.

Args:
*args: The values to generate a key for.
**kwargs: The columns and values to generate a key for.

Returns:
The generated key as bytes.
"""
if not args and not kwargs:
raise ValueError("At least one value must be provided")

if args and kwargs:
raise ValueError("Cannot mix positional and keyword arguments in make_key")

if args and not len(args) == 1 and not isinstance(args[0], list):
raise ValueError("When using positional arguments, provide a single list of values")

return self.index.make_key(args[0] if args else kwargs)

def search(self, *args: RecordValue, **kwargs: RecordValue) -> Record:
"""Search the index for the requested values.

Searching modifies the cursor state. Searching again will search from the current position.
Reset the cursor with :meth:`reset` to start from the beginning.

Args:
*args: The values to search for.
**kwargs: The columns and values to search for.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the found record.
"""
key = self.index.make_key(kwargs)
key = self.make_key(*args, **kwargs)
return self.search_key(key, exact=True)

def search_key(self, key: bytes, exact: bool = True) -> Record:
Expand All @@ -88,24 +138,27 @@ def search_key(self, key: bytes, exact: bool = True) -> Record:
next record that is greater than or equal to the key.
"""
self._primary.search(key, exact)
return self._record()
return self.record()

def seek(self, **kwargs: RecordValue) -> None:
def seek(self, *args: RecordValue, **kwargs: RecordValue) -> Self:
"""Seek to the record with the given values.

Args:
*args: The values to seek to.
**kwargs: The columns and values to seek to.
"""
key = self.index.make_key(kwargs)
key = self.make_key(*args, **kwargs)
self.search_key(key, exact=False)
return self

def seek_key(self, key: bytes) -> None:
def seek_key(self, key: bytes) -> Self:
"""Seek to the record with the given ``key``.

Args:
key: The key to seek to.
"""
self._primary.search(key, exact=False)
return self

def find(self, **kwargs: RecordValue) -> Record | None:
"""Find a record in the index.
Expand All @@ -130,7 +183,10 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
other_columns = kwargs

# We need at least an exact match on the indexed columns
self.search(**indexed_columns)
try:
self.search(**indexed_columns)
except KeyNotFoundError:
return

current_key = self._primary.node().key

Expand All @@ -150,7 +206,7 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
if current_key != self._primary.node().key:
break

record = self._record()
record = self.record()
for k, v in other_columns.items():
value = record.get(k)
# If the record value is a list, we do a check based on the queried value
Expand All @@ -172,39 +228,3 @@ def find_all(self, **kwargs: RecordValue) -> Iterator[Record]:
self._primary.next()
except NoNeighbourPageError:
break

def record(self) -> Record:
"""Return the record the cursor is currently on.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the current record.
"""
return self._record()

def next(self) -> Record:
"""Move the cursor to the next record and return it.

Can move the cursor to the next page as a side effect.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the next record.
"""
try:
self._primary.next()
except NoNeighbourPageError:
raise IndexError("No next record")
return self._record()

def prev(self) -> Record:
"""Move the cursor to the previous node and return it.

Can move the cursor to the previous page as a side effect.

Returns:
A :class:`~dissect.database.ese.record.Record` object of the previous record.
"""
try:
self._primary.prev()
except NoNeighbourPageError:
raise IndexError("No previous record")
return self._record()
1 change: 1 addition & 0 deletions dissect/database/ese/ese.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ def __init__(self, fh: BinaryIO, impacket_compat: bool = False):
self.fh = fh
self.impacket_compat = impacket_compat

self.fh.seek(0)
self.header = c_ese.DBFILEHDR(fh)
if self.header.ulMagic != ulDAEMagic:
raise InvalidDatabase("invalid file header signature")
Expand Down
Loading