[wip] experiment: hist table format

db/kv/mdbx/kv_mdbx.go

@@ -988,6 +988,17 @@ func (tx *MdbxTx) DropTable(bucket string) error {
 	return tx.dropEvenIfBucketIsNotDeprecated(bucket)
 }
 
+// DropTableForced drops a DBI and all its data unconditionally, regardless of IsDeprecated status.
+// Intended for use in schema migrations where table flags need to change (e.g. adding/removing DupSort).
+// After dropping, the table is re-created with the current config flags on the next DB open.
+func DropTableForced(tx kv.RwTx, name string) error {
+	mdbxTx, ok := tx.(*MdbxTx)
+	if !ok {
+		return fmt.Errorf("DropTableForced: expected *MdbxTx, got %T", tx)
+	}
+	return mdbxTx.dropEvenIfBucketIsNotDeprecated(name)
+}
+
 func (tx *MdbxTx) ExistsTable(bucket string) (bool, error) {
 	if cfg, ok := tx.db.buckets[bucket]; ok {
 		return cfg.DBI != NonExistingDBI, nil

db/kv/tables.go

@@ -144,35 +144,35 @@ const (
 	// Domains/History/InvertedIndices
 	// Constants have "Tbl" prefix, to avoid collision with actual Domain names
 	// This constants is very rarely used in APP, but Domain/History/Idx names are widely used
-	TblAccountVals        = "AccountVals"
-	TblAccountHistoryKeys = "AccountHistoryKeys"
-	TblAccountHistoryVals = "AccountHistoryVals"
-	TblAccountIdx         = "AccountIdx"
-
-	TblStorageVals        = "StorageVals"
-	TblStorageHistoryKeys = "StorageHistoryKeys"
-	TblStorageHistoryVals = "StorageHistoryVals"
-	TblStorageIdx         = "StorageIdx"
-
-	TblCodeVals        = "CodeVals"
-	TblCodeHistoryKeys = "CodeHistoryKeys"
-	TblCodeHistoryVals = "CodeHistoryVals"
-	TblCodeIdx         = "CodeIdx"
-
-	TblCommitmentVals        = "CommitmentVals"
-	TblCommitmentHistoryKeys = "CommitmentHistoryKeys"
-	TblCommitmentHistoryVals = "CommitmentHistoryVals"
-	TblCommitmentIdx         = "CommitmentIdx"
-
-	TblReceiptVals        = "ReceiptVals"
-	TblReceiptHistoryKeys = "ReceiptHistoryKeys"
-	TblReceiptHistoryVals = "ReceiptHistoryVals"
-	TblReceiptIdx         = "ReceiptIdx"
-
-	TblRCacheVals        = "ReceiptCacheVals"
-	TblRCacheHistoryKeys = "ReceiptCacheHistoryKeys"
-	TblRCacheHistoryVals = "ReceiptCacheHistoryVals"
-	TblRCacheIdx         = "ReceiptCacheIdx"
+	TblAccountVals          = "AccountVals"
+	TblAccountHistoryData   = "AccountHistoryKeys" // DataTable: txNum+key → prev_value (non-DupSort, sequential writes)
+	TblAccountHistoryInvIdx = "AccountHistoryVals" // InvIndexTable: key → txNum (DupSort, no embedded value)
+	TblAccountIdx           = "AccountIdx"
+
+	TblStorageVals          = "StorageVals"
+	TblStorageHistoryData   = "StorageHistoryKeys"
+	TblStorageHistoryInvIdx = "StorageHistoryVals"
+	TblStorageIdx           = "StorageIdx"
+
+	TblCodeVals          = "CodeVals"
+	TblCodeHistoryData   = "CodeHistoryKeys"
+	TblCodeHistoryInvIdx = "CodeHistoryVals"
+	TblCodeIdx           = "CodeIdx"
+
+	TblCommitmentVals          = "CommitmentVals"
+	TblCommitmentHistoryData   = "CommitmentHistoryKeys"
+	TblCommitmentHistoryInvIdx = "CommitmentHistoryVals"
+	TblCommitmentIdx           = "CommitmentIdx"
+
+	TblReceiptVals          = "ReceiptVals"
+	TblReceiptHistoryData   = "ReceiptHistoryKeys"
+	TblReceiptHistoryInvIdx = "ReceiptHistoryVals"
+	TblReceiptIdx           = "ReceiptIdx"
+
+	TblRCacheVals          = "ReceiptCacheVals"
+	TblRCacheHistoryData   = "ReceiptCacheHistoryKeys"
+	TblRCacheHistoryInvIdx = "ReceiptCacheHistoryVals"
+	TblRCacheIdx           = "ReceiptCacheIdx"
 
 	TblLogAddressKeys = "LogAddressKeys"
 	TblLogAddressIdx  = "LogAddressIdx"
@@ -186,7 +186,7 @@ const (
 
 	// Prune progress of execution: tableName -> [8bytes of invStep]latest pruned key
 	// Could use table constants `Tbl{Account,Storage,Code,Commitment}Keys` for domains
-	// corresponding history tables `Tbl{Account,Storage,Code,Commitment}HistoryKeys` for history
+	// corresponding history tables `Tbl{Account,Storage,Code,Commitment}HistoryData` for history
 	// and `Tbl{Account,Storage,Code,Commitment}Idx` for inverted indices
 	TblPruningProgress = "PruningProgress"
 	// tableName -> txTo;last pruned val
@@ -342,33 +342,33 @@ var ChaindataTables = []string{
 	BorWitnesses,
 	BorWitnessSizes,
 	TblAccountVals,
-	TblAccountHistoryKeys,
-	TblAccountHistoryVals,
+	TblAccountHistoryData,
+	TblAccountHistoryInvIdx,
 	TblAccountIdx,
 
 	TblStorageVals,
-	TblStorageHistoryKeys,
-	TblStorageHistoryVals,
+	TblStorageHistoryData,
+	TblStorageHistoryInvIdx,
 	TblStorageIdx,
 
 	TblCodeVals,
-	TblCodeHistoryKeys,
-	TblCodeHistoryVals,
+	TblCodeHistoryData,
+	TblCodeHistoryInvIdx,
 	TblCodeIdx,
 
 	TblCommitmentVals,
-	TblCommitmentHistoryKeys,
-	TblCommitmentHistoryVals,
+	TblCommitmentHistoryData,
+	TblCommitmentHistoryInvIdx,
 	TblCommitmentIdx,
 
 	TblReceiptVals,
-	TblReceiptHistoryKeys,
-	TblReceiptHistoryVals,
+	TblReceiptHistoryData,
+	TblReceiptHistoryInvIdx,
 	TblReceiptIdx,
 
 	TblRCacheVals,
-	TblRCacheHistoryKeys,
-	TblRCacheHistoryVals,
+	TblRCacheHistoryData,
+	TblRCacheHistoryInvIdx,
 	TblRCacheIdx,
 
 	TblLogAddressKeys,
@@ -521,31 +521,33 @@ var ChaindataTablesCfg = TableCfg{
 		DupToLen:                  28,
 	},
 
-	TblAccountVals:        {Flags: DupSort},
-	TblAccountHistoryKeys: {Flags: DupSort},
-	TblAccountHistoryVals: {Flags: DupSort},
-	TblAccountIdx:         {Flags: DupSort},
-
-	TblStorageVals:        {Flags: DupSort},
-	TblStorageHistoryKeys: {Flags: DupSort},
-	TblStorageHistoryVals: {Flags: DupSort},
-	TblStorageIdx:         {Flags: DupSort},
-
-	TblCodeHistoryKeys: {Flags: DupSort},
-	TblCodeIdx:         {Flags: DupSort},
-
-	TblCommitmentVals:        {Flags: DupSort},
-	TblCommitmentHistoryKeys: {Flags: DupSort},
-	TblCommitmentHistoryVals: {Flags: DupSort},
-	TblCommitmentIdx:         {Flags: DupSort},
-
-	TblReceiptVals:        {Flags: DupSort},
-	TblReceiptHistoryKeys: {Flags: DupSort},
-	TblReceiptHistoryVals: {Flags: DupSort},
-	TblReceiptIdx:         {Flags: DupSort},
-
-	TblRCacheHistoryKeys: {Flags: DupSort},
-	TblRCacheIdx:         {Flags: DupSort},
+	TblAccountVals:          {Flags: DupSort},
+	TblAccountHistoryData:   {},               // DataTable: non-DupSort (txNum+key → value, sequential writes)
+	TblAccountHistoryInvIdx: {Flags: DupSort}, // InvIndexTable: key → txNum
+	TblAccountIdx:           {Flags: DupSort},
+
+	TblStorageVals:          {Flags: DupSort},
+	TblStorageHistoryData:   {},
+	TblStorageHistoryInvIdx: {Flags: DupSort},
+	TblStorageIdx:           {Flags: DupSort},
+
+	TblCodeHistoryData:   {},               // was DupSort (TblCodeHistoryKeys)
+	TblCodeHistoryInvIdx: {Flags: DupSort}, // was non-DupSort (TblCodeHistoryVals)
+	TblCodeIdx:           {Flags: DupSort},
+
+	TblCommitmentVals:          {Flags: DupSort},
+	TblCommitmentHistoryData:   {},
+	TblCommitmentHistoryInvIdx: {Flags: DupSort},
+	TblCommitmentIdx:           {Flags: DupSort},
+
+	TblReceiptVals:          {Flags: DupSort},
+	TblReceiptHistoryData:   {},
+	TblReceiptHistoryInvIdx: {Flags: DupSort},
+	TblReceiptIdx:           {Flags: DupSort},
+
+	TblRCacheHistoryData:   {},               // was DupSort (TblRCacheHistoryKeys)
+	TblRCacheHistoryInvIdx: {Flags: DupSort}, // was non-DupSort (TblRCacheHistoryVals)
+	TblRCacheIdx:           {Flags: DupSort},
 
 	TblLogAddressKeys: {Flags: DupSort},
 	TblLogAddressIdx:  {Flags: DupSort},

db/migrations/history_table_format_change.go

@@ -0,0 +1,48 @@
+// Copyright 2024 The Erigon Authors
+// This file is part of Erigon.
+//
+// Erigon is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Erigon is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with Erigon. If not, see <http://www.gnu.org/licenses/>.
+
+package migrations
+
+import (
+	"context"
+
+	"github.com/erigontech/erigon/common/dir"
+	"github.com/erigontech/erigon/common/log/v3"
+	"github.com/erigontech/erigon/db/datadir"
+	"github.com/erigontech/erigon/db/kv"
+)
+
+// HistoryTableFormatChange removes chaindata so it gets rebuilt with the new history table format:
+//   - DataTable (formerly HistoryKeys): txNum+key → prev_value (non-DupSort, sequential writes)
+//   - InvIndexTable (formerly HistoryVals): key → txNum (DupSort, no embedded value)
+var HistoryTableFormatChange = Migration{
+	Name: "history_table_format_change",
+	Up: func(db kv.RwDB, dirs datadir.Dirs, progress []byte, BeforeCommit Callback, logger log.Logger) (err error) {
+		tx, err := db.BeginRw(context.Background())
+		if err != nil {
+			return err
+		}
+		defer tx.Rollback()
+
+		if err := BeforeCommit(tx, nil, true); err != nil {
+			return err
+		}
+		if err := dir.RemoveAll(dirs.Chaindata); err != nil {
+			return err
+		}
+		return nil
+	},
+}

db/migrations/migrations.go

@@ -53,6 +53,7 @@ var migrations = map[kv.Label][]Migration{
 	dbcfg.ChainDB: {
 		dbSchemaVersion5,
 		ResetStageTxnLookup,
+		HistoryTableFormatChange,
 	},
 	dbcfg.TxPoolDB: {},
 	dbcfg.SentryDB: {},

db/rawdb/rawdbhelpers/rawdbhelpers.go

@@ -23,8 +23,8 @@ import (
 )
 
 func IdxStepsCountV3(tx kv.Tx, stepSize uint64) float64 {
-	fst, _ := kv.FirstKey(tx, kv.TblAccountHistoryKeys)
-	lst, _ := kv.LastKey(tx, kv.TblAccountHistoryKeys)
+	fst, _ := kv.FirstKey(tx, kv.TblAccountHistoryData)
+	lst, _ := kv.LastKey(tx, kv.TblAccountHistoryData)
 	if len(fst) > 0 && len(lst) > 0 {
 		fstTxNum := binary.BigEndian.Uint64(fst)
 		lstTxNum := binary.BigEndian.Uint64(lst)

db/state/domain_test.go

@@ -1481,7 +1481,6 @@ func TestDomain_GetAfterAggregation(t *testing.T) {
 	require.NoError(err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressNone         //seg.CompressKeys | seg.CompressVals
 	d.FilenameBase = kv.CommitmentDomain.String()
@@ -1556,7 +1555,6 @@ func TestDomainRange(t *testing.T) {
 	require.NoError(err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressNone // seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressNone         // seg.CompressKeys | seg.CompressVals
 	d.FilenameBase = kv.AccountsDomain.String()
@@ -1674,7 +1672,6 @@ func TestDomain_CanScanPruneAfterAggregation(t *testing.T) {
 	require.NoError(t, err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressKeys | seg.CompressVals
 	d.FilenameBase = kv.CommitmentDomain.String()
@@ -1771,7 +1768,6 @@ func TestDomain_CanHashPruneAfterAggregation(t *testing.T) {
 	require.NoError(t, err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressKeys | seg.CompressVals
 	d.FilenameBase = kv.CommitmentDomain.String()
@@ -1869,7 +1865,6 @@ func TestDomain_PruneAfterAggregation(t *testing.T) {
 	require.NoError(t, err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressNone //seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressNone         //seg.CompressKeys | seg.CompressVals
 
@@ -2016,7 +2011,6 @@ func TestDomain_PruneProgress(t *testing.T) {
 	require.NoError(t, err)
 	defer rwTx.Rollback()
 
-	d.HistoryLargeValues = false
 	d.History.Compression = seg.CompressKeys | seg.CompressVals
 	d.Compression = seg.CompressKeys | seg.CompressVals
 
@@ -2542,7 +2536,6 @@ func TestDomainContext_findShortenedKey(t *testing.T) {
 	require.NoError(t, err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = true
 	domainRoTx := d.BeginFilesRo()
 	defer domainRoTx.Close()
 	writer := domainRoTx.NewWriter()
@@ -2624,7 +2617,6 @@ func TestCanBuild(t *testing.T) {
 	require.NoError(t, err)
 	defer tx.Rollback()
 
-	d.HistoryLargeValues = true
 	domainRoTx := d.BeginFilesRo()
 	defer domainRoTx.Close()
 
@@ -2662,24 +2654,23 @@ func TestTraceKey_SmallVals(t *testing.T) {
 	if testing.Short() {
 		t.Skip("slow test")
 	}
-	testTraceKey(t, false)
+	testTraceKey(t)
 }
 
 func TestTraceKey_LargeVals(t *testing.T) {
 	if testing.Short() {
 		t.Skip("slow test")
 	}
-	testTraceKey(t, true)
+	testTraceKey(t)
 }
 
-func testTraceKey(t *testing.T, largeVals bool) {
+func testTraceKey(t *testing.T) {
 	logger := log.New()
 	logEvery := time.NewTicker(30 * time.Second)
 	defer logEvery.Stop()
 	ctx := context.Background()
 
 	db, d := testDbAndDomain(t, logger)
-	d.HistoryLargeValues = largeVals
 
 	txs := fillDomain(t, d, db, logger)
 	err := db.UpdateNosync(ctx, func(tx kv.RwTx) error {

db/state/gc_test.go

@@ -113,14 +113,9 @@ func TestGCReadAfterRemoveFile(t *testing.T) {
 			require.Nil(lastOnFs.decompressor)
 		})
 	}
-	t.Run("large_values", func(t *testing.T) {
-		db, h, txs := filledHistory(t, true, logger)
-		test(t, h, db, txs)
-	})
-	t.Run("small_values", func(t *testing.T) {
-		db, h, txs := filledHistory(t, false, logger)
-		test(t, h, db, txs)
-	})
+
+	db, h, txs := filledHistory(t, logger)
+	test(t, h, db, txs)
 }
 
 func TestDomainGCReadAfterRemoveFile(t *testing.T) {