blockchain, netsync: implement a complete headers-first download during ibd

blockchain/accept.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/btcsuite/btcd/btcutil"
 	"github.com/btcsuite/btcd/database"
+	"github.com/btcsuite/btcd/wire"
 )
 
 // maybeAcceptBlock potentially accepts a block into the block chain and, if
@@ -63,11 +64,20 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags)
 	// Create a new block node for the block and add it to the node index. Even
 	// if the block ultimately gets connected to the main chain, it starts out
 	// on a side chain.
-	blockHeader := &block.MsgBlock().Header
-	newNode := newBlockNode(blockHeader, prevNode)
-	newNode.status = statusDataStored
-
-	b.index.AddNode(newNode)
+	//
+	// If a header-only node already exists (from maybeAcceptBlockHeader),
+	// upgrade its status rather than creating a new node.  Creating a new
+	// node would overwrite the index entry, orphaning the pointer held by
+	// bestHeader's chainView and breaking Contains checks.
+	newNode := b.index.LookupNode(block.Hash())
+	if newNode != nil {
+		b.index.SetStatusFlags(newNode, statusDataStored)
+	} else {
+		blockHeader := &block.MsgBlock().Header
+		newNode = newBlockNode(blockHeader, prevNode)
+		newNode.status = statusDataStored | statusHeaderStored
+		b.index.AddNode(newNode)
+	}
 	err = b.index.flushToDB()
 	if err != nil {
 		return false, err
@@ -92,3 +102,135 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags)
 
 	return isMainChain, nil
 }
+
+// maybeAcceptBlockHeader potentially accepts the header to the block index and,
+// if accepted, returns a bool indicating if the header extended the best chain
+// of headers.  It also performs several context independent checks as well as
+// those which depend on its position within the header chain.
+//
+// The flags are passed to CheckBlockHeaderSanity and CheckBlockHeaderContext
+// which allow the skipping of PoW check or the check for the block difficulty,
+// median time check, and the BIP94 check.
+//
+// The skipCheckpoint boolean allows skipping of the check for if the header is
+// part of the existing checkpoints.
+//
+// In the case the block header is already known, the associated block node is
+// examined to determine if the block is already known to be invalid, in which
+// case an appropriate error will be returned.
+//
+// This function MUST be called with the chain lock held (for writes).
+func (b *BlockChain) maybeAcceptBlockHeader(header *wire.BlockHeader,
+	flags BehaviorFlags, skipCheckpoint bool) (bool, error) {
+
+	// Orphan headers are not allowed and this function should never be called
+	// with the genesis block.
+	prevHash := &header.PrevBlock
+	prevNode := b.index.LookupNode(prevHash)
+	if prevNode == nil {
+		str := fmt.Sprintf("previous block %s is not known", prevHash)
+		return false, ruleError(ErrPreviousBlockUnknown, str)
+	}
+
+	// This header is invalid if its previous node is invalid.
+	if b.index.NodeStatus(prevNode).KnownInvalid() {
+		str := fmt.Sprintf(
+			"previous block %s is known to be invalid", prevHash)
+		return false, ruleError(ErrInvalidAncestorBlock, str)
+	}
+
+	// Avoid validating the header again if its validation status is already
+	// known.  Invalid headers are never added to the block index, so if there
+	// is an entry for the block hash, the header itself is known to be valid.
+	hash := header.BlockHash()
+	node := b.index.LookupNode(&hash)
+	if node != nil {
+		nodeStatus := b.index.NodeStatus(node)
+		if nodeStatus&statusValidateFailed != 0 {
+			str := fmt.Sprintf("block %s is known to be invalid", hash)
+			return false, ruleError(ErrKnownInvalidBlock, str)
+		} else if nodeStatus&statusInvalidAncestor != 0 {
+			str := fmt.Sprintf("block %s has an invalid ancestor", hash)
+			return false, ruleError(ErrInvalidAncestorBlock, str)
+		}
+
+		// If the node is in the bestHeaders chainview, it's in the main chain.
+		// If it isn't, then we'll go through the verification process below.
+		if b.bestHeader.Contains(node) {
+			return true, nil
+		}
+	}
+
+	// Perform context-free sanity checks on the block header.
+	err := CheckBlockHeaderSanity(
+		header, b.chainParams.PowLimit, b.timeSource, flags)
+	if err != nil {
+		return false, err
+	}
+
+	// The block must pass all of the validation rules which depend on the
+	// position of the block within the block chain.
+	err = CheckBlockHeaderContext(header, prevNode, flags, b, skipCheckpoint)
+	if err != nil {
+		return false, err
+	}
+
+	// Create a new block node for the block and add it to the block index.
+	//
+	// Note that the additional information for the actual transactions and
+	// witnesses in the block can't be populated until the full block data is
+	// known since that information is not available in the header.
+	if node == nil {
+		node = newBlockNode(header, prevNode)
+		node.status = statusHeaderStored
+		b.index.AddNode(node)
+	}
+
+	// Flush the block index to database at this point since we added the
+	// node.
+	err = b.index.flushToDB()
+	if err != nil {
+		return false, err
+	}
+
+	// Check if the header extends the best header tip.
+	isMainChain := false
+	parentHash := &header.PrevBlock
+	if parentHash.IsEqual(&b.bestHeader.Tip().hash) {
+		log.Debugf("accepted header %v as the new header tip", node.hash)
+
+		// This header is now the end of the best headers.
+		b.bestHeader.SetTip(node)
+		isMainChain = true
+		return isMainChain, nil
+	}
+
+	// We're extending (or creating) a side chain, but the cumulative
+	// work for this new side chain is not enough to make it the new chain.
+	if node.workSum.Cmp(b.bestHeader.Tip().workSum) <= 0 {
+		// Log information about how the header is forking the chain.
+		fork := b.bestHeader.FindFork(node)
+		if fork.hash.IsEqual(parentHash) {
+			log.Infof("FORK: BlockHeader %v(%v) forks the chain at block %v(%v) "+
+				"but did not have enough work to be the "+
+				"main chain", node.hash, node.height, fork.hash, fork.height)
+		} else {
+			log.Infof("EXTEND FORK: BlockHeader %v(%v) extends a side chain "+
+				"which forks the chain at block %v(%v)",
+				node.hash, node.height, fork.hash, fork.height)
+		}
+
+		return false, nil
+	}
+
+	prevTip := b.bestHeader.Tip()
+	log.Infof("NEW BEST HEADER CHAIN: BlockHeader %v(%v) is now a longer "+
+		"PoW chain than the previous header tip of %v(%v).",
+		node.hash, node.height,
+		prevTip.hash, prevTip.height)
+
+	b.bestHeader.SetTip(node)
+	isMainChain = true
+
+	return isMainChain, nil
+}

blockchain/accept_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2013-2026 The btcsuite developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package blockchain
+
+import (
+	"testing"
+
+	"github.com/btcsuite/btcd/blockchain/internal/testhelper"
+	"github.com/btcsuite/btcd/btcutil"
+)
+
+// TestMaybeAcceptBlockReusesHeaderNode ensures that when a block header is
+// processed first via ProcessBlockHeader and later the full block arrives via
+// ProcessBlock, the existing blockNode pointer is reused rather than replaced.
+// Replacing the pointer would orphan the entry held by bestHeader's chainView,
+// causing bestHeader.Contains(index.LookupNode(hash)) to return false and
+// breaking IsValidHeader and downstream netsync checks.
+func TestMaybeAcceptBlockReusesHeaderNode(t *testing.T) {
+	chain, params, tearDown := utxoCacheTestChain(
+		"TestMaybeAcceptBlockReusesHeaderNode")
+	defer tearDown()
+
+	// Build a base chain of 3 blocks.
+	//
+	// genesis -> 1 -> 2 -> 3
+	tip := btcutil.NewBlock(params.GenesisBlock)
+	_, _, err := addBlocks(3, chain, tip, []*testhelper.SpendableOut{})
+	if err != nil {
+		t.Fatalf("failed to build base chain: %v", err)
+	}
+
+	// Create block 4 without processing it.
+	prevBlock, err := chain.BlockByHeight(3)
+	if err != nil {
+		t.Fatalf("failed to get block at height 3: %v", err)
+	}
+	block4, _, err := newBlock(chain, prevBlock, nil)
+	if err != nil {
+		t.Fatalf("failed to create block 4: %v", err)
+	}
+
+	// Process block 4's header first.
+	block4Hash := block4.Hash()
+	_, err = chain.ProcessBlockHeader(
+		&block4.MsgBlock().Header, BFNone, false)
+	if err != nil {
+		t.Fatalf("ProcessBlockHeader fail: %v", err)
+	}
+
+	// Capture the header-only node pointer from the index.
+	headerNode := chain.index.LookupNode(block4Hash)
+	if headerNode == nil {
+		t.Fatal("header node not found in block index")
+	}
+
+	// Now process the full block.
+	_, _, err = chain.ProcessBlock(block4, BFNone)
+	if err != nil {
+		t.Fatalf("ProcessBlock fail: %v", err)
+	}
+
+	// The index must still hold the same pointer that bestHeader has.
+	// Before the fix, maybeAcceptBlock would create a fresh node and
+	// overwrite the index entry, orphaning the pointer in bestHeader.
+	fullBlockNode := chain.index.LookupNode(block4Hash)
+	if fullBlockNode != headerNode {
+		t.Fatal("ProcessBlock replaced the header node pointer " +
+			"instead of reusing it")
+	}
+	if !chain.bestHeader.Contains(fullBlockNode) {
+		t.Fatal("node no longer in bestHeader after ProcessBlock")
+	}
+}

blockchain/blockindex.go

@@ -33,6 +33,9 @@ const (
 	// has failed validation, thus the block is also invalid.
 	statusInvalidAncestor
 
+	// statusHeaderStored indicates that the block's header is stored on disk.
+	statusHeaderStored
+
 	// statusNone indicates that the block has no validation state flags set.
 	//
 	// NOTE: This must be defined last in order to avoid influencing iota.
@@ -46,6 +49,11 @@ func (status blockStatus) HaveData() bool {
 	return status&statusDataStored != 0
 }
 
+// HaveHeader returns whether the header data is stored in the database.
+func (status blockStatus) HaveHeader() bool {
+	return status&statusHeaderStored != 0
+}
+
 // KnownValid returns whether the block is known to be valid. This will return
 // false for a valid block that has not been fully validated yet.
 func (status blockStatus) KnownValid() bool {
@@ -377,14 +385,16 @@ func newBlockIndex(db database.DB, chainParams *chaincfg.Params) *blockIndex {
 	}
 }
 
-// HaveBlock returns whether or not the block index contains the provided hash.
+// HaveBlock returns whether or not the block index contains the provided hash
+// and if the data exists on disk.
 //
 // This function is safe for concurrent access.
 func (bi *blockIndex) HaveBlock(hash *chainhash.Hash) bool {
 	bi.RLock()
-	_, hasBlock := bi.index[*hash]
+	node, hasBlock := bi.index[*hash]
+	haveData := hasBlock && node.status.HaveData()
 	bi.RUnlock()
-	return hasBlock
+	return haveData
 }
 
 // LookupNode returns the block node identified by the provided hash.  It will
@@ -497,8 +507,37 @@ func (bi *blockIndex) flushToDB() error {
 		return nil
 	}
 
+	// Check if any dirty node actually needs to be written.  Header-only
+	// nodes are skipped for backwards compatibility (see NOTE below), so
+	// if every dirty node is header-only, we can avoid opening a write
+	// transaction entirely.  This matters during header sync where every
+	// ProcessBlockHeader call would otherwise open a no-op write txn.
+	needsWrite := false
+	for node := range bi.dirty {
+		if node.status.HaveData() {
+			needsWrite = true
+			break
+		}
+	}
+	if !needsWrite {
+		bi.dirty = make(map[*blockNode]struct{})
+		bi.Unlock()
+		return nil
+	}
+
 	err := bi.db.Update(func(dbTx database.Tx) error {
 		for node := range bi.dirty {
+			// NOTE: we specifically don't flush the block indexes that
+			// we don't have the data for backwards compatibility.
+			// While flushing would save us the work of re-downloading
+			// the block headers upon restart, if the user were to start
+			// up a btcd node with an older version, it would result in
+			// an unrecoverable error as older versions would consider a
+			// blockNode being present as having the block data as well.
+			if node.status.HaveHeader() &&
+				!node.status.HaveData() {
+				continue
+			}
 			err := dbStoreBlockNode(dbTx, node)
 			if err != nil {
 				return err