Skip to content

Fix endianness of delete on rewind table #1080

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
quantumagi merged 4 commits into from
Oct 30, 2022
Merged

Fix endianness of delete on rewind table #1080

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8e04eb2
Fix endianness of delete on rewind table
quantumagi Oct 27, 2022
efe436f
Fix duplicate call
quantumagi Oct 30, 2022
8ab0e84
Fix consistency
quantumagi Oct 30, 2022
0b611cc
Update mock
quantumagi Oct 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 61 additions & 10 deletions src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,24 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using LiteDB;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using NBitcoin;
using Stratis.Bitcoin.Configuration;
using Stratis.Bitcoin.Consensus;
using Stratis.Bitcoin.Consensus.Rules;
using Stratis.Bitcoin.Controllers.Models;
using Stratis.Bitcoin.Database;
using Stratis.Bitcoin.Features.BlockStore.AddressIndexing;
using Stratis.Bitcoin.Features.BlockStore.Repositories;
using Stratis.Bitcoin.Features.Consensus.CoinViews;
using Stratis.Bitcoin.Features.Consensus.Rules;
using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Networks;
using Stratis.Bitcoin.Primitives;
using Stratis.Bitcoin.Tests.Common;
Expand All @@ -26,30 +35,49 @@ public class AddressIndexerTests

private readonly Mock<IConsensusManager> consensusManagerMock;

private readonly ChainIndexer chainIndexer;

private readonly Network network;

private readonly IConsensusRuleEngine consensusRuleEngine;

private readonly ChainedHeader genesisHeader;

public AddressIndexerTests()
{
this.network = new StraxMain();
this.chainIndexer = new ChainIndexer(this.network);
var nodeSettings = new NodeSettings(this.network, args: new[] { "-addressindex", "-txindex" });

var mockingServices = new ServiceCollection()
.AddSingleton(this.network)
.AddSingleton(new StoreSettings(NodeSettings.Default(this.network))
{
AddressIndex = true,
TxIndex = true
})
.AddSingleton(nodeSettings)
.AddSingleton(nodeSettings.LoggerFactory)
.AddSingleton(new DataFolder(TestBase.CreateTestDir(this)))
.AddSingleton(new ChainIndexer(this.network))
.AddSingleton<IScriptAddressReader, ScriptAddressReader>()
.AddSingleton<ConsensusRulesContainer>()
.AddSingleton<IConsensusRuleEngine, PosConsensusRuleEngine>()
.AddSingleton<IBlockRepository>(typeof(BlockRepository<LevelDb>).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder))))
.AddSingleton<IBlockStore, BlockStoreQueue>()
.AddSingleton<ICoindb>(typeof(Coindb<LevelDb>).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder))))
.AddSingleton<ICoinView, CachedCoinView>()
.AddSingleton(this.chainIndexer)
.AddSingleton<IDateTimeProvider, DateTimeProvider>()
.AddSingleton<IAddressIndexer, AddressIndexer>();

var mockingContext = new MockingContext(mockingServices);

this.addressIndexer = mockingContext.GetService<IAddressIndexer>();
this.genesisHeader = mockingContext.GetService<ChainIndexer>().GetHeader(0);

var rulesContainer = mockingContext.GetService<ConsensusRulesContainer>();
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(LoadCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SaveCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(StraxCoinviewRule)) as FullValidationConsensusRule);
rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SetActivationDeploymentsFullValidationRule)) as FullValidationConsensusRule);

this.consensusManagerMock = mockingContext.GetService<Mock<IConsensusManager>>();
this.consensusRuleEngine = mockingContext.GetService<IConsensusRuleEngine>();
}

[Fact]
Expand All @@ -64,7 +92,7 @@ public void CanInitializeAndDispose()
[Fact]
public void CanIndexAddresses()
{
List<ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network);
List<ChainedHeader> headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network, chainIndexer: this.chainIndexer);
this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headers.Last());

Script p2pk1 = this.GetRandomP2PKScript(out string address1);
Expand Down Expand Up @@ -106,7 +134,7 @@ public void CanIndexAddresses()
tx.Inputs.Add(new TxIn(new OutPoint(block5.Transactions.First().GetHash(), 0)));
var block10 = new Block() { Transactions = new List<Transaction>() { tx } };

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
ChainedHeaderBlock GetChainedHeaderBlock(uint256 hash)
{
ChainedHeader header = headers.SingleOrDefault(x => x.HashBlock == hash);

Expand All @@ -123,8 +151,26 @@ public void CanIndexAddresses()
}

return new ChainedHeaderBlock(new Block(), header);
}

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
{
return GetChainedHeaderBlock(hash);
});

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<List<uint256>>())).Returns((List<uint256> hashes) =>
{
return hashes.Select(h => GetChainedHeaderBlock(h)).ToArray();
});

this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny<ChainedHeader>(), It.IsAny<int>(), It.IsAny<CancellationTokenSource>())).Returns((ChainedHeader header, int size, CancellationTokenSource token) =>
{
return headers.Where(h => h.Height > header.Height).Select(h => GetChainedHeaderBlock(h.HashBlock)).ToArray();
});

this.consensusManagerMock.Setup(x => x.ConsensusRules).Returns(this.consensusRuleEngine);

this.consensusRuleEngine.Initialize(headers.Last(), this.consensusManagerMock.Object);
this.addressIndexer.Initialize();

TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headers.Last());
Expand All @@ -137,7 +183,7 @@ public void CanIndexAddresses()
// Now trigger rewind to see if indexer can handle reorgs.
ChainedHeader forkPoint = headers.Single(x => x.Height == 8);

List<ChainedHeader> headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network);
List<ChainedHeader> headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network, chainIndexer: this.chainIndexer);

this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny<uint256>())).Returns((uint256 hash) =>
{
Expand All @@ -146,6 +192,11 @@ public void CanIndexAddresses()
return new ChainedHeaderBlock(new Block(), headerFork);
});

this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny<ChainedHeader>(), It.IsAny<int>(), It.IsAny<CancellationTokenSource>())).Returns((ChainedHeader header, int size, CancellationTokenSource token) =>
{
return headersFork.Where(h => h.Height > header.Height).Select(h => new ChainedHeaderBlock(new Block(), h)).ToArray();
});

this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headersFork.Last());
TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headersFork.Last());

Expand Down
2 changes: 1 addition & 1 deletion src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ private async Task IndexAddressesContinuouslyAsync()
if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader)
blockToProcess = prefetchedBlock.Block;
else
blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock).Block;
blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock)?.Block;

if (blockToProcess == null)
{
Expand Down
72 changes: 43 additions & 29 deletions src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
using System.Threading;
using Microsoft.Extensions.Logging;
using NBitcoin;
using Stratis.Bitcoin.Configuration;
using Stratis.Bitcoin.Configuration.Settings;
using Stratis.Bitcoin.Consensus;
using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders;
using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules;
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Primitives;
using Stratis.Bitcoin.Utilities;
using Stratis.Bitcoin.Utilities.Extensions;
using TracerAttributes;
Expand Down Expand Up @@ -128,16 +129,16 @@ public long GetScriptSize
private DateTime lastCacheFlushTime;
private readonly Network network;
private readonly IDateTimeProvider dateTimeProvider;
private readonly IBlockStore blockStore;
private readonly CancellationTokenSource cancellationToken;
private IConsensusManager consensusManager;
private readonly ConsensusSettings consensusSettings;
private readonly ChainIndexer chainIndexer;
private CachePerformanceSnapshot latestPerformanceSnapShot;

private readonly Random random;

public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer,
StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null)
public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer,
StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null)
{
Guard.NotNull(coindb, nameof(CachedCoinView.coindb));

Expand All @@ -149,7 +150,6 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim
this.chainIndexer = chainIndexer;
this.stakeChainStore = stakeChainStore;
this.rewindDataIndexCache = rewindDataIndexCache;
this.blockStore = blockStore;
this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping);
this.lockobj = new object();
this.cachedUtxoItems = new Dictionary<OutPoint, CacheItem>();
Expand Down Expand Up @@ -178,50 +178,55 @@ public void Sync()

Flush();

ChainedHeader fork = this.chainIndexer[coinViewTip.Hash];
if (fork == null)
if (coinViewTip.Height > this.chainIndexer.Height || this.chainIndexer[coinViewTip.Hash] == null)
{
// Determine the last usable height.
int height = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1;
fork = this.chainIndexer[(height < 0) ? coinViewTip.Height : height];
}
// The coinview tip is above the chain height or on a fork.
// Determine the first unusable height by finding the first rewind data that is not on the consensus chain.
int unusableHeight = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || (this.GetRewindData(h)?.PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock), 2, coinViewTip.Height - 1);
ChainedHeader fork = this.chainIndexer[unusableHeight - 2];

while (coinViewTip.Height != fork.Height)
{
if ((coinViewTip.Height % 100) == 0)
this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork);
while (coinViewTip.Height != fork.Height)
{
if ((coinViewTip.Height % 100) == 0)
this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork);

// If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
// The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
coinViewTip = this.coindb.Rewind(new HashHeightPair(fork));
};

this.blockHash = coinViewTip;
this.innerBlockHash = this.blockHash;
}

// If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip.
// The node will complete loading before connecting to peers so the chain will never know that a reorg happened.
coinViewTip = this.coindb.Rewind(new HashHeightPair(fork));
};
CatchUp();
}
}

public void Initialize(IConsensusManager consensusManager)
private void CatchUp()
{
ChainedHeader chainTip = this.chainIndexer.Tip;

this.coindb.Initialize();

Sync();

HashHeightPair coinViewTip = this.coindb.GetTipHash();

// If the coin view is behind the block store then catch up from the block store.
if (coinViewTip.Height < chainTip.Height)
{
try
{
IConsensusRuleEngine consensusRuleEngine = consensusManager.ConsensusRules;
IConsensusRuleEngine consensusRuleEngine = this.consensusManager.ConsensusRules;

var loadCoinViewRule = consensusRuleEngine.GetRule<LoadCoinviewRule>();
var saveCoinViewRule = consensusRuleEngine.GetRule<SaveCoinviewRule>();
var coinViewRule = consensusRuleEngine.GetRule<CoinViewRule>();
var deploymentsRule = consensusRuleEngine.GetRule<SetActivationDeploymentsFullValidationRule>();

foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(this.chainIndexer[coinViewTip.Hash], this.chainIndexer, this.cancellationToken, batchSize: 1000))
foreach (ChainedHeaderBlock chb in this.consensusManager.GetBlocksAfterBlock(this.chainIndexer[coinViewTip.Hash], 1000, this.cancellationToken))
{
ChainedHeader chainedHeader = chb?.ChainedHeader;
if (chainedHeader == null)
break;

Block block = chb.Block;
if (block == null)
break;

Expand All @@ -244,7 +249,7 @@ public void Initialize(IConsensusManager consensusManager)
coinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();

// Saves the changes to the coinview.
saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
finally
Expand All @@ -258,6 +263,15 @@ public void Initialize(IConsensusManager consensusManager)
}
}
}
}

public void Initialize(IConsensusManager consensusManager)
{
this.consensusManager = consensusManager;

this.coindb.Initialize();

Sync();

this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash());
}
Expand Down Expand Up @@ -485,7 +499,7 @@ public void Flush(bool force = true)
this.cachedRewindData.Clear();
this.rewindDataSizeBytes = 0;
this.dirtyCacheCount = 0;
this.innerBlockHash = this.blockHash;
this.innerBlockHash = this.blockHash;
this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow();
}
}
Expand Down
Loading