Enable coin view to "catch up" if forked or below chain tip

src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs

@@ -2,11 +2,14 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using Microsoft.Extensions.Logging;
 using NBitcoin;
 using Stratis.Bitcoin.Configuration.Settings;
 using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders;
+using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
+using Stratis.Bitcoin.Interfaces;
 using Stratis.Bitcoin.Utilities;
 using Stratis.Bitcoin.Utilities.Extensions;
 using TracerAttributes;
@@ -126,13 +129,15 @@ public long GetScriptSize
         private readonly Network network;
         private readonly ICheckpoints checkpoints;
         private readonly IDateTimeProvider dateTimeProvider;
+        private readonly IBlockStore blockStore;
+        private readonly CancellationTokenSource cancellationToken;
         private readonly ConsensusSettings consensusSettings;
         private CachePerformanceSnapshot latestPerformanceSnapShot;
-        private int lastCheckpointHeight;
 
         private readonly Random random;
 
-        public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null)
+        public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, 
+            StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null)
         {
             Guard.NotNull(coindb, nameof(CachedCoinView.coindb));
 
@@ -144,41 +149,110 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb,
             this.consensusSettings = consensusSettings;
             this.stakeChainStore = stakeChainStore;
             this.rewindDataIndexCache = rewindDataIndexCache;
+            this.blockStore = blockStore;
+            this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping);
             this.lockobj = new object();
             this.cachedUtxoItems = new Dictionary<OutPoint, CacheItem>();
             this.performanceCounter = new CachePerformanceCounter(this.dateTimeProvider);
             this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow();
             this.cachedRewindData = new Dictionary<int, RewindData>();
             this.random = new Random();
 
-            this.lastCheckpointHeight = this.checkpoints.GetLastCheckpointHeight();
-
             this.MaxCacheSizeBytes = consensusSettings.MaxCoindbCacheInMB * 1024 * 1024;
             this.CacheFlushTimeIntervalSeconds = consensusSettings.CoindbIbdFlushMin * 60;
 
             if (nodeStats.DisplayBenchStats)
                 nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300);
         }
 
-        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer)
+        /// <summary>
+        /// Remain on-chain.
+        /// </summary>
+        public void Sync(ChainIndexer chainIndexer)
+        {
+            lock (this.lockobj)
+            {
+                HashHeightPair coinViewTip = this.GetTipHash();
+                if (coinViewTip.Hash == chainIndexer.Tip.HashBlock)
+                    return;
+
+                Flush();
+
+                ChainedHeader fork = chainIndexer[coinViewTip.Hash];
+                if (fork == null)
+                {
+                    // Determine the last usable height.
+                    int height = BinarySearch.BinaryFindFirst(h => (h > chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1;
+                    fork = chainIndexer[(height < 0) ? coinViewTip.Height : height];
+                }
+
+                while (coinViewTip.Height != fork.Height)
+                {
+                    if ((coinViewTip.Height % 100) == 0)
+                        this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork);
+
+                    // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
+                    // The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
+                    coinViewTip = this.coindb.Rewind(new HashHeightPair(fork));
+                };
+            }
+        }
+
+        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine)
         {
             this.coindb.Initialize(chainTip);
 
+            Sync(chainIndexer);
+
             HashHeightPair coinViewTip = this.coindb.GetTipHash();
 
-            while (true)
+            // If the coin view is behind the block store then catch up from the block store.
+            if (coinViewTip.Height < chainTip.Height)
             {
-                ChainedHeader pendingTip = chainTip.FindAncestorOrSelf(coinViewTip.Hash);
+                try
+                {
+                    var loadCoinViewRule = consensusRuleEngine.GetRule<LoadCoinviewRule>();
+                    var saveCoinViewRule = consensusRuleEngine.GetRule<SaveCoinviewRule>();
+                    var coinViewRule = consensusRuleEngine.GetRule<CoinViewRule>();
+                    var deploymentsRule = consensusRuleEngine.GetRule<SetActivationDeploymentsFullValidationRule>();
+
+                    foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(chainIndexer[coinViewTip.Hash], chainIndexer, this.cancellationToken, batchSize: 1000))
+                    {
+                        if (block == null)
+                            break;
+
+                        if ((chainedHeader.Height % 10000) == 0)
+                        {
+                            this.Flush(true);
+                            this.logger.LogInformation("Rebuilding coin view from '{0}' to {1}.", chainedHeader, chainTip);
+                        }
+
+                        var utxoRuleContext = consensusRuleEngine.CreateRuleContext(new ValidationContext() { ChainedHeaderToValidate = chainedHeader, BlockToValidate = block });
+                        utxoRuleContext.SkipValidation = true;
 
-                if (pendingTip != null)
-                    break;
+                        // Set context flags.
+                        deploymentsRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
 
-                if ((coinViewTip.Height % 100) == 0)
-                    this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, chainTip);
+                        // Loads the coins spent by this block into utxoRuleContext.UnspentOutputSet.
+                        loadCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
 
-                // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
-                // The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
-                coinViewTip = this.coindb.Rewind(new HashHeightPair(chainTip));
+                        // Spends the coins.
+                        coinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
+
+                        // Saves the changes to the coinview.
+                        saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
+                    }
+                }
+                finally
+                {
+                    this.Flush(true);
+
+                    if (this.cancellationToken.IsCancellationRequested)
+                    {
+                        this.logger.LogDebug("Rebuilding cancelled due to application stopping.");
+                        throw new OperationCanceledException();
+                    }
+                }
             }
 
             this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash());

src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using NBitcoin;
+using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Utilities;
 
 namespace Stratis.Bitcoin.Features.Consensus.CoinViews
@@ -15,7 +16,8 @@ public interface ICoinView
         /// </summary>
         /// <param name="chainTip">The chain tip.</param>
         /// <param name="chainIndexer">The chain indexer.</param>
-        void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer);
+        /// <param name="consensusRuleEngine">The consensus rule engine.</param>
+        void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine);
 
         /// <summary>
         /// Retrieves the block hash of the current tip of the coinview.
@@ -40,6 +42,12 @@ public interface ICoinView
         /// <param name="rewindDataList">List of rewind data items to be persisted. This should only be used when calling <see cref="DBreezeCoinView.SaveChanges" />.</param>
         void SaveChanges(IList<UnspentOutput> unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List<RewindData> rewindDataList = null);
 
+        /// <summary>
+        /// Brings the coinview back on-chain if a re-org occurred.
+        /// </summary>
+        /// <param name="chainIndexer">The current consensus chain.</param>
+        void Sync(ChainIndexer chainIndexer);
+
         /// <summary>
         /// Obtains information about unspent outputs.
         /// </summary>

src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using NBitcoin;
+using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Utilities;
 using ReaderWriterLock = NBitcoin.ReaderWriterLock;
 
@@ -33,11 +34,16 @@ public InMemoryCoinView(HashHeightPair tipHash)
         }
 
         /// <inheritdoc />
-        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer)
+        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine)
         {
             throw new NotImplementedException();
         }
 
+        /// <inheritdoc />
+        public void Sync(ChainIndexer chainIndexer)
+        {
+        }
+
         /// <inheritdoc />
         public HashHeightPair GetTipHash()
         {

src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs

@@ -25,6 +25,7 @@ public override Task RunAsync(RuleContext context)
             // unless the coinview threshold is reached.
             this.Logger.LogDebug("Saving coinview changes.");
             var utxoRuleContext = context as UtxoRuleContext;
+            this.PowParent.UtxoSet.Sync(this.Parent.ChainIndexer);
             this.PowParent.UtxoSet.SaveChanges(utxoRuleContext.UnspentOutputSet.GetCoins(), new HashHeightPair(oldBlock), new HashHeightPair(nextBlock));
 
             return Task.CompletedTask;
@@ -66,7 +67,7 @@ public override Task RunAsync(RuleContext context)
         {
             // Check that the current block has not been reorged.
             // Catching a reorg at this point will not require a rewind.
-            if (context.ValidationContext.BlockToValidate.Header.HashPrevBlock != this.Parent.ChainState.ConsensusTip.HashBlock)
+            if (context.ValidationContext.BlockToValidate.Header.HashPrevBlock != this.PowParent.UtxoSet.GetTipHash().Hash)
             {
                 this.Logger.LogDebug("Reorganization detected.");
                 ConsensusErrors.InvalidPrevTip.Throw();

src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs

@@ -67,7 +67,7 @@ public override void Initialize(ChainedHeader chainTip)
         {
             base.Initialize(chainTip);
 
-            this.UtxoSet.Initialize(chainTip, this.ChainIndexer);
+            this.UtxoSet.Initialize(chainTip, this.ChainIndexer, this);
         }
 
         public override async Task<ValidationContext> FullValidationAsync(ChainedHeader header, Block block)

src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using NBitcoin;
+using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Features.Consensus;
 using Stratis.Bitcoin.Features.Consensus.CoinViews;
 using Stratis.Bitcoin.Features.MemoryPool.Interfaces;
@@ -49,11 +50,16 @@ public MempoolCoinView(Network network, ICoinView inner, ITxMempool memPool, Sch
             this.Set = new UnspentOutputSet();
         }
 
-        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer)
+        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine)
         {
             throw new NotImplementedException();
         }
 
+        /// <inheritdoc />
+        public void Sync(ChainIndexer chainIndexer)
+        {
+        }
+
         /// <summary>
         /// Gets the unspent transaction output set.
         /// </summary>

src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using NBitcoin;
+using Stratis.Bitcoin.Consensus;
 using Stratis.Bitcoin.Features.Consensus.CoinViews;
 using Stratis.Bitcoin.Utilities;
 using ReaderWriterLock = NBitcoin.ReaderWriterLock;
@@ -34,11 +35,16 @@ public TestInMemoryCoinView(HashHeightPair tipHash)
         }
 
         /// <inheritdoc />
-        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer)
+        public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine)
         {
             throw new NotImplementedException();
         }
 
+        /// <inheritdoc />
+        public void Sync(ChainIndexer chainIndexer)
+        {
+        }
+
         /// <inheritdoc />
         public HashHeightPair GetTipHash()
         {