diff --git a/.gitignore b/.gitignore index 19864c6..af33ab9 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,5 @@ __recovery/ # Castalia statistics file (since XE7 Castalia is distributed with Delphi) *.stat +* .idea +* diff --git a/FirebirdRepairBase.dpr b/FirebirdRepairBase.dpr index cdaec90..2a7359f 100644 --- a/FirebirdRepairBase.dpr +++ b/FirebirdRepairBase.dpr @@ -15,7 +15,8 @@ uses uPageAnalyzer in 'uPageAnalyzer.pas', uBtreePage in 'uBtreePage.pas', uDatabaseStats in 'uDatabaseStats.pas', - uFlagManager in 'uFlagManager.pas'; + uFlagManager in 'uFlagManager.pas', + uRecordParser in 'uRecordParser.pas'; {$R *.res} diff --git a/FirebirdRepairBase.dproj b/FirebirdRepairBase.dproj index 6f9a953..e0d0b30 100644 --- a/FirebirdRepairBase.dproj +++ b/FirebirdRepairBase.dproj @@ -122,6 +122,7 @@ + Base diff --git a/README.md b/README.md index 07d0002..85771d4 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,84 @@ # FirebirdRepairBase -Program for analize and repare broken firebird database, it is working on the low level -This is dirty version of program, now program is in develop process. - -Now it features: -* check database pages by type -* replace some header page params -* set READ ONLY and FORCE WRITE flags -* replace check sum on pages -* generate new page - -# In developing -* Analisys pages, pump data. -* Analisys TIP sequence, generate lost TIP pages - -# Firebird Database Pages -* Database Header Page -* Page Inventory Page (PIP) -* Transaction Inventory Page (TIP) -* Pointer Page -* Data Page -* Index Root Page -* Index B-Tree Page -* Blob Data Page -* Generator Page -* Write Ahead Log Page + +A low-level tool for analyzing and potentially recovering data from **broken** Firebird database files (`.fdb`, `.gdb`). It bypasses the standard Firebird engine to read raw page structures and data. + +**Warning:** This is a **development version**. It directly manipulates database files. Always create a **backup** of your database before using this tool. Use at your own risk. + +## Features + +* **Database Structure Analysis:** + * Opens and reads Firebird database files. + * Identifies and counts pages by type (Header, PIP, TIP, Pointer, Data, Index Root/BTree, Blob, Generator, WAL). + * Displays basic page information (number, type, checksum, relation ID). +* **Header Manipulation:** + * Reads and interprets database header flags. + * Allows setting/clearing specific flags (e.g., Read Only, Force Write). +* **Page Content Inspection:** + * Reads specific pages by number. + * Extracts and displays raw record fragments from Data Pages. + * Allows changing page type and checksum (Dangerous!). +* **Transaction Analysis:** + * Reads Transaction Inventory Pages (TIP). + * Counts transactions by state (Active, Limbo, Dead, Committed). +* **Basic Data Recovery:** + * Extracts raw data fragments from Data Pages. + * *(WIP)* Basic MVCC (Multi-Version Concurrency Control) awareness for data visibility (see TODO). + * *(New)* Parses basic record headers to identify deleted records and data length. +* **Record Fragment Parsing:** + * Introduces `uRecordParser` module for analyzing individual record fragments. + * Extracts key information like transaction ID, back pointer, flags, and data length from the record header. + +## In Development (TODO) + +* **Advanced Data Recovery:** + * Implement full MVCC logic to reconstruct the correct, visible version of records based on transaction states and TIP page data. + * Export recovered data to a new, valid Firebird database or other formats (e.g., CSV). + * Analyze and potentially repair B-Tree index structures. + * Analyze and potentially repair Blob page chains. +* **Deeper Integrity Checks:** + * Verify logical links between pages (e.g., Pointer Pages -> Data Pages, Record Back Pointers). + * Identify orphaned pages. +* **Page Generation/Repair:** + * Implement logic to generate missing TIP or PIP pages if necessary. + * Attempt to rebuild damaged page structures. +* **User Interface Improvements:** + * Add search functionality within page data. + * Provide more detailed views for different page types (B-Tree keys, Blob data). + * Add a 'Restore from Backup' feature. + * Improve error handling and user feedback. + +## Supported Firebird Pages + +* Database Header Page +* Page Inventory Page (PIP) +* Transaction Inventory Page (TIP) +* Pointer Page +* Data Page +* Index Root Page +* Index B-Tree Page +* Blob Data Page +* Generator Page +* Write Ahead Log Page (WAL) (Note: WAL functionality might be limited) + +## Technical Details + +* **Language:** Pascal (Delphi). +* **Target:** Windows. +* **Approach:** Direct file I/O, parsing raw page structures according to Firebird Internal Format documentation (`fbint-page-1.html`) and Firebird source code conventions. +* **Modules:** + * `uStructs`: Core data structures (TPag, THdr, etc.). + * `uPageAnalyzer`: General page reading and type identification. + * `uDataPage`: Data Page header parsing and record fragment extraction. + * `uTipPage`: TIP page header parsing and transaction state analysis. + * `uBtreePage`: B-Tree page header parsing and key extraction. + * `uRecordParser`: *(New)* Parses headers of individual record fragments. + * `uDatabaseStats`: Aggregates statistics using `uPageAnalyzer`. + * `uFlagManager`: Manages database header flags. ## License -MIT: + +MIT ## Author + Gordienko Roman diff --git a/main.pas b/main.pas index eceb210..fb8236e 100644 --- a/main.pas +++ b/main.pas @@ -5,9 +5,9 @@ interface uses Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms, Dialogs, StdCtrls, FileCtrl, ComCtrls, uStructs, DB, ExtCtrls, - Menus, uCommon, uDatabase, XPMan, uDataPage, Grids, DBGrids, DBClient, + Menus, uCommon, XPMan, uDataPage, Grids, DBGrids, DBClient, DBCtrls, Spin, Vcl.Buttons, - uPageAnalyzer, uDatabaseStats, uFlagManager; + uPageAnalyzer, uDatabaseStats, uFlagManager, StrUtils; type TfrmMain = class(TForm) @@ -200,6 +200,7 @@ procedure TfrmMain.btnReWriteClick(Sender: TObject); PageInfo: TPageInfo; NewPageBuffer: TBytes; FileStream: TFileStream; + ModifiedHeader: TPag; // --- ��������� --- begin // ReWriting page - ��������: ��� ������� ��������! if Application.MessageBox('Do You Want to Rewrite Database Page? THIS IS DANGEROUS!', @@ -214,10 +215,18 @@ procedure TfrmMain.btnReWriteClick(Sender: TObject); PageInfo := FPageAnalyzer.GetPageInfo(PageNum); NewPageBuffer := Copy(PageInfo.Buffer); // �������� ������� ����� - // ������������ ��������� � ������ - TPag(NewPageBuffer[0]).pag_checksum := UShort(NewChecksum); + // --- �����������: ���������� Move ��� ����������� ����������� � ��������� ��������� --- + // ��������� ����������� ��������� �� ������ � ��������� ���������� + Move(NewPageBuffer[0], ModifiedHeader, SizeOf(TPag)); + + // ������� ���� � ��������� ���������� + ModifiedHeader.pag_checksum := UShort(NewChecksum); if (NewType >= 0) and (NewType <= $FF) then // ��������� �������� - TPag(NewPageBuffer[0]).pag_type := SChar(NewType); + ModifiedHeader.pag_type := SChar(NewType); + + // ��������� ���������� ��������� ���������� ������� � ����� + Move(ModifiedHeader, NewPageBuffer[0], SizeOf(TPag)); + // --- ����� ����������� --- // ���������� ���������� ����� ������� � ���� FileStream := TFileStream.Create(FPageAnalyzer.FileName, fmOpenReadWrite or fmShareExclusive); @@ -290,8 +299,7 @@ procedure TfrmMain.btnGetHeaderFlagsClick(Sender: TObject); try Flags := FFlagManager.GetFlags; - // ��������� ������ ��� ����������� (��������, �������� ������������� ��� ������ ������) - // ��� ��������, ������� ������ �������� ������ + edtFlags.text := 'Active flags: '; if Flags.ActiveShadow then edtFlags.text := edtFlags.text + 'ActiveShadow, '; if Flags.ForceWrite then edtFlags.text := edtFlags.text + 'ForceWrite, '; @@ -299,11 +307,11 @@ procedure TfrmMain.btnGetHeaderFlagsClick(Sender: TObject); if Flags.NoReserve then edtFlags.text := edtFlags.text + 'NoReserve, '; if Flags.SqlDialect3 then edtFlags.text := edtFlags.text + 'SqlDialect3, '; if Flags.ReadOnly then edtFlags.text := edtFlags.text + 'ReadOnly, '; - // ������� ��������� ������� - if edtFlags.text.EndsWith(', ') then - edtFlags.text := edtFlags.text.Remove(edtFlags.text.Length - 2); - // ��� ����� ������� ��� ����� � ��������� ������� + if EndsStr(', ', edtFlags.text) then // ���������� EndsStr �� StrUtils + edtFlags.text := Copy(edtFlags.text, 1, Length(edtFlags.text) - 2); // ������� ��������� 2 ������� + + lstDBFlags.Items.Add(Format('Active Shadow: %s', [BoolToStr(Flags.ActiveShadow, True)])); lstDBFlags.Items.Add(Format('Force Write: %s', [BoolToStr(Flags.ForceWrite, True)])); lstDBFlags.Items.Add(Format('No Checksums: %s', [BoolToStr(Flags.NoChecksums, True)])); @@ -348,20 +356,20 @@ procedure TfrmMain.checkDB; lstLog.Items.Add('Database page size: ' + IntToStr(FPageAnalyzer.PageSize)); lstLog.Items.Add('Scanning pages...'); - // ��������� ���������� ����� FDatabaseStats + FDatabaseStats.CalculateStats; - // �������� ���������� + PageStats := FDatabaseStats.PageStats; TxStats := FDatabaseStats.TransactionStats; RelStats := FDatabaseStats.RelationStats; - // ��������� �������� ��� (��� �����������, ���� ������ ��� ������) + pbDetectedPage.Min := 0; - pbDetectedPage.Max := 100; // ������� + pbDetectedPage.Max := 100; pbDetectedPage.Position := 100; - // ������� ���������� �������� + lstLog.Items.Add('--- Page Type Summary ---'); lstLog.Items.Add('Count Header Pages: ' + IntToStr(PageStats.HeaderPages)); lstLog.Items.Add('Count Page Inventory Pages (PIP): ' + IntToStr(PageStats.PipPages)); @@ -369,7 +377,7 @@ procedure TfrmMain.checkDB; lstLog.Items.Add('Count Pointer Pages: ' + IntToStr(PageStats.PointerPages)); lstLog.Items.Add('Count Data Pages: ' + IntToStr(PageStats.DataPages)); lstLog.Items.Add('Count Index Root Pages: ' + IntToStr(PageStats.IndexRootPages)); - lstLog.Items.Add('Count Index B-Tree Pages: ' + IntToStr(PageStats.IndexBtreePages)); // ���������� + lstLog.Items.Add('Count Index B-Tree Pages: ' + IntToStr(PageStats.IndexBtreePages)); lstLog.Items.Add('Count Blob Pages: ' + IntToStr(PageStats.BlobPages)); lstLog.Items.Add('Count Generator Pages: ' + IntToStr(PageStats.GeneratorPages)); lstLog.Items.Add('Count Write Ahead Log Pages: ' + IntToStr(PageStats.WalPages)); @@ -415,12 +423,10 @@ procedure TfrmMain.btnWriteFlagsClick(Sender: TObject); try NewFlags := FFlagManager.GetFlags; - // ��������� ����� � ������������ � ���������� + NewFlags.ForceWrite := chkSetFW.Checked; NewFlags.ReadOnly := chkReadOnly.Checked; - // �������� ������ ����� �� ������������� - // ������������� ����� ����� FFlagManager FFlagManager.SetFlags(NewFlags); Application.MessageBox('Header flags updated!', 'Information', MB_OK + MB_ICONINFORMATION); @@ -452,14 +458,14 @@ procedure TfrmMain.UpdateFlagCheckboxesFromManager; end; end; -// tsFlagsShow - ������ ���������� UpdateFlagCheckboxesFromManager + procedure TfrmMain.tsFlagsShow(Sender: TObject); begin - // ��������� �������� ��� ������ ������� + UpdateFlagCheckboxesFromManager; end; -// pgcServicesChange - ��������� MaxValue ��� sePosition + procedure TfrmMain.pgcServicesChange(Sender: TObject); begin if tsGenerateNewPage.Showing then @@ -513,15 +519,13 @@ procedure TfrmMain.btnGotoPageClick(Sender: TObject); end; btnGotoPage.Enabled := False; - mmoData.Clear; // ������� Memo ����� ������� - - // ���������� ������� �� uDataPage ��� ���������� ���������� + mmoData.Clear; PageFragments := ExtractDataFragments(PageInfo.Buffer, PageInfo.Size); pbDataProgress.Min := 0; - pbDataProgress.Max := Length(PageFragments) - 1; // ���������� ���������� + pbDataProgress.Max := Length(PageFragments) - 1; - for i := 0 to High(PageFragments) do // ���������� High ��� ������� + for i := 0 to High(PageFragments) do begin pbDataProgress.Position := i; mmoData.Lines.Add('Fragment ' + IntToStr(i) + ', Offset: ' + IntToStr(PageFragments[i].Offset)); diff --git a/uBtreePage.pas b/uBtreePage.pas index 88f192f..94ac709 100644 --- a/uBtreePage.pas +++ b/uBtreePage.pas @@ -3,7 +3,7 @@ interface uses - Windows, SysUtils, uStructs; + Windows, SysUtils, uStructs, Math; const pag_left = $01; // ��� ����� ����� �������� � ������ @@ -16,7 +16,6 @@ interface key_dupeq = $08; // ���� ������� ��� �������� (��� ���������� ��������) type - // ������������� ����� ��������� B-Tree �������� TBtreePageHeader = packed record pag_header: TPag; // 0x00-0x0F - ����������� ��������� pag_next: SLong; // 0x10-0x13 - ��������� �� ��������� �������� (����) @@ -28,18 +27,18 @@ interface // pag_keys ���������� � ����� 0x20 end; - // ��� ��� ������������� ������ ����� + TBtreeKey = record Length: UShort; Flags: SChar; - Data: TBytes; // ������ ����� - PageOrRecordNumber: SLong; // ��� ���������� ����� - ����� ��������, ��� ������� - ����� ������ + Data: TBytes; + PageOrRecordNumber: SLong; end; - // ��� ��� ������� ������ + TBtreeKeyArray = array of TBtreeKey; - // ����� ��� ������� B-Tree �������� + TBtreePageAnalyzer = class private FPageSize: Integer; @@ -47,8 +46,12 @@ TBtreePageAnalyzer = class FHeader: TBtreePageHeader; FPageData: TBytes; // ������ ���� �������� (��� ���������� ������) - // ���������� ������� ��� ��������� ����� �� �������� + function GetKeyAtOffset(Offset: Integer): TBtreeKey; + + function GetIsLeaf: Boolean; + function GetIsRoot: Boolean; + function GetIsLeftmost: Boolean; public // ����������� - ��������� ����� ��������, ������ �������� � ����� �������� constructor Create(const PageBuffer: TBytes; PageSize: Integer; PageNum: ULong); @@ -67,11 +70,11 @@ TBtreePageAnalyzer = class // �������� ����� ������������ �������� property ParentPage: SLong read FHeader.pag_parent; // ��� �������� ��������? - property IsLeaf: Boolean read (FHeader.pag_flags and pag_leaf) <> 0; + property IsLeaf: Boolean read GetIsLeaf; // ���������� �����-������ // ��� �������� ��������? - property IsRoot: Boolean read (FHeader.pag_flags and pag_root) <> 0; + property IsRoot: Boolean read GetIsRoot; // ���������� �����-������ // ��� ����� ����� ��������? - property IsLeftmost: Boolean read (FHeader.pag_flags and pag_left) <> 0; + property IsLeftmost: Boolean read GetIsLeftmost; // ���������� �����-������ end; @@ -107,32 +110,31 @@ function TBtreePageAnalyzer.GetKeyAtOffset(Offset: Integer): TBtreeKey; KeyFlags: SChar; KeyDataOffset, KeyPageOffset: Integer; begin - // ��������� ������� + if (Offset + 2 {key_length} + 1 {key_flags} > Length(FPageData)) then raise Exception.Create('Invalid offset for key in B-Tree page.'); - // ������ key_length (2 �����) KeyLength := PWord(@FPageData[Offset])^; Inc(Offset, 2); - // ������ key_flags (1 ����) + KeyFlags := PShortInt(@FPageData[Offset])^; Inc(Offset, 1); - // ���������, �� ������� �� ����� ������ �� ������� + if (Offset + KeyLength + 4 {key_page} > Length(FPageData)) then raise Exception.Create('Key length exceeds page boundary.'); - // �������� key_data + SetLength(Result.Data, KeyLength); if KeyLength > 0 then Move(FPageData[Offset], Result.Data[0], KeyLength); Inc(Offset, KeyLength); - // ������ key_page (4 �����) + Result.PageOrRecordNumber := PLongInt(@FPageData[Offset])^; - // ��������� ���� ���������� + Result.Length := KeyLength; Result.Flags := KeyFlags; end; @@ -145,19 +147,32 @@ function TBtreePageAnalyzer.GetKeys: TBtreeKeyArray; begin SetLength(Keys, FHeader.pag_count); - // �������� ������ ������� ������ + CurrentOffset := SizeOf(TBtreePageHeader); for i := 0 to FHeader.pag_count - 1 do begin Keys[i] := GetKeyAtOffset(CurrentOffset); - - // ��������� �������� ���������� ����� - // key_length(2) + key_flags(1) + key_data(key_length) + key_page(4) CurrentOffset := CurrentOffset + 2 + 1 + Keys[i].Length + 4; end; Result := Keys; end; + +function TBtreePageAnalyzer.GetIsLeaf: Boolean; +begin + Result := (FHeader.pag_flags and pag_leaf) <> 0; +end; + +function TBtreePageAnalyzer.GetIsRoot: Boolean; +begin + Result := (FHeader.pag_flags and pag_root) <> 0; +end; + +function TBtreePageAnalyzer.GetIsLeftmost: Boolean; +begin + Result := (FHeader.pag_flags and pag_left) <> 0; +end; + end. diff --git a/uDatabaseStats.pas b/uDatabaseStats.pas index 3b20eee..baf1670 100644 --- a/uDatabaseStats.pas +++ b/uDatabaseStats.pas @@ -162,23 +162,27 @@ procedure TDatabaseStats.CalculateRelationStats; var PageNum: ULong; PageInfo: TPageInfo; - DataPageHeader: TData_Page_Fixed_Part; + DataPageHeader: TData_Page_Fixed_Part; // ��� �� uDataPage Fragments: TRecordFragmentsArray; FoundRelation: Boolean; i, j: Integer; begin - SetLength(FRelationStats, 0); + // �������������� ������ ���������� �� ���������� + SetLength(FRelationStats, 0); // ���� ������ for PageNum := 0 to FPageAnalyzer.GetLastPageNumber do begin PageInfo := FPageAnalyzer.GetPageInfo(PageNum); - if PageInfo.Header.pag_type = pag_data then + if PageInfo.Header.pag_type = pag_data then // pag_data = $05 begin + // ���������, ���������� �� ������ ��� ������������� ����� ��������� Data Page if Length(PageInfo.Buffer) >= SizeOf(TData_Page_Fixed_Part) then begin - DataPageHeader := TData_Page_Fixed_Part(PageInfo.Buffer[0]); + // --- �����������: ���������� Move ������ ������� ���������� ���� --- + Move(PageInfo.Buffer[0], DataPageHeader, SizeOf(TData_Page_Fixed_Part)); var RelationID := DataPageHeader.dpg_relation; + // ������ ������ ��� ����� RelationID � ������� ���������� FoundRelation := False; for j := 0 to High(FRelationStats) do begin @@ -190,23 +194,24 @@ procedure TDatabaseStats.CalculateRelationStats; end; end; - + // ���� ��������� �� ���� � �������, ������� ��� if not FoundRelation then begin SetLength(FRelationStats, Length(FRelationStats) + 1); FRelationStats[High(FRelationStats)].RelationID := RelationID; FRelationStats[High(FRelationStats)].PageCount := 1; - FRelationStats[High(FRelationStats)].RecordCount := 0; + FRelationStats[High(FRelationStats)].RecordCount := 0; // ���� 0 end; - - Fragments := ExtractDataFragments(PageInfo.Buffer, PageInfo.Size); - + // ������: ������� ������� (����������) �� �������� + // ��������� ��������� + Fragments := ExtractDataFragments(PageInfo.Buffer, PageInfo.Size); // ExtractDataFragments �� uDataPage + // ������ ������ ��������� ����� ��� ���������� RecordCount for j := 0 to High(FRelationStats) do begin if FRelationStats[j].RelationID = RelationID then begin - Inc(FRelationStats[j].RecordCount, Length(Fragments)); + Inc(FRelationStats[j].RecordCount, Length(Fragments)); // ���������� ���������� ���������� Break; end; end; @@ -215,6 +220,7 @@ procedure TDatabaseStats.CalculateRelationStats; end; end; + procedure TDatabaseStats.CalculateStats; begin CalculatePageStats;