From 56861185333c36fb0d2951f89019b6b5150d4899 Mon Sep 17 00:00:00 2001 From: zeptin Date: Wed, 23 Mar 2022 16:40:21 +0200 Subject: [PATCH 01/44] Added two RPC methods for listing transactions (#941) (cherry picked from commit d846b7eba11faf788445921b9b0c8c6ccf52a0c6) --- .../Models/TransactionInfoModel.cs | 79 ++++++++++++++++++ .../Models/TransactionsSinceBlockModel.cs | 17 ++++ .../RPCClient.Wallet.cs | 23 +++++- .../Interfaces/IWalletManager.cs | 6 ++ .../WalletManager.cs | 15 ++++ .../WalletRPCController.cs | 80 +++++++++++++++++++ .../RPC/RpcBitcoinImmutableTests.cs | 17 ++++ 7 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.RPC/Models/TransactionInfoModel.cs create mode 100644 src/Stratis.Bitcoin.Features.RPC/Models/TransactionsSinceBlockModel.cs diff --git a/src/Stratis.Bitcoin.Features.RPC/Models/TransactionInfoModel.cs b/src/Stratis.Bitcoin.Features.RPC/Models/TransactionInfoModel.cs new file mode 100644 index 0000000000..b176c4fc2e --- /dev/null +++ b/src/Stratis.Bitcoin.Features.RPC/Models/TransactionInfoModel.cs @@ -0,0 +1,79 @@ +using NBitcoin; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Stratis.Bitcoin.Features.RPC.Models +{ + public class TransactionInfoModel + { + public TransactionInfoModel() + { + } + + public TransactionInfoModel(JObject tx) + { + if (tx["blockheight"] != null) + { + this.BlockHeight = (int)tx["blockheight"]; + } + + this.BlockHash = (string)tx["blockhash"]; + decimal amount = (decimal)tx["amount"]; + this.Amount = new Money((long)(amount * Money.COIN)); + this.Confirmations = (int)tx["confirmations"]; + this.TxId = (string)tx["txid"]; + this.BlockIndex = (int)tx["blockindex"]; + + if (tx["generated"] != null) + { + this.Generated = (bool)tx["generated"]; + } + else + { + // Default to True for earlier versions, i.e. if not present + this.Generated = false; + } + } + + /// + /// The transaction id. + [JsonProperty(Order = 1, PropertyName = "txid")] + public string TxId { get; set; } + + /// + /// The index of the transaction in the block that includes it. + /// + [JsonProperty(PropertyName = "blockindex")] + public int BlockIndex { get; set; } + + /// + /// Only present if transaction only input is a coinbase one. + /// + [JsonProperty(PropertyName = "generated")] + public bool Generated { get; set; } + + /// + /// The transaction amount. + /// + [JsonProperty(PropertyName = "amount")] + public long Amount { get; set; } + + /// + /// The number of confirmations. + /// + [JsonProperty(PropertyName = "confirmations")] + public int Confirmations { get; set; } + + /// + /// The hash of the block containing this transaction. + /// + [JsonProperty(PropertyName = "blockhash")] + public string BlockHash { get; set; } + + /// + /// The height of the block containing this transaction. + /// + [JsonProperty(PropertyName = "blockheight")] + public int BlockHeight { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.RPC/Models/TransactionsSinceBlockModel.cs b/src/Stratis.Bitcoin.Features.RPC/Models/TransactionsSinceBlockModel.cs new file mode 100644 index 0000000000..ffe7c50df9 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.RPC/Models/TransactionsSinceBlockModel.cs @@ -0,0 +1,17 @@ +using Newtonsoft.Json; + +namespace Stratis.Bitcoin.Features.RPC.Models +{ + public class TransactionsSinceBlockModel + { + /// + /// All transactions. + [JsonProperty(Order = 1, PropertyName = "transactions")] + public TransactionInfoModel[] Transactions { get; set; } + + /// + /// The hash of the block since which we got transactions. + [JsonProperty(PropertyName = "lastblock")] + public string LastBlock { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.RPC/RPCClient.Wallet.cs b/src/Stratis.Bitcoin.Features.RPC/RPCClient.Wallet.cs index 4f9bd73b78..d2cca535f9 100644 --- a/src/Stratis.Bitcoin.Features.RPC/RPCClient.Wallet.cs +++ b/src/Stratis.Bitcoin.Features.RPC/RPCClient.Wallet.cs @@ -40,8 +40,8 @@ wallet listaddressgroupings Yes wallet listlockunspent wallet listreceivedbyaccount wallet listreceivedbyaddress - wallet listsinceblock - wallet listtransactions + wallet listsinceblock Yes + wallet listtransactions Yes wallet listunspent Yes wallet lockunspent Yes wallet move @@ -495,6 +495,25 @@ public IEnumerable ListSecrets() } } + /// + /// Returns an array of transactions to this wallet from a specified block. + /// + public TransactionsSinceBlockModel ListSinceBlock(string blockhash = "", int targetConfirmations = 1) + { + RPCResponse response = SendCommand(RPCOperations.listsinceblock, blockhash, targetConfirmations); + + return response.Result.ToObject(); + } + + /// + /// Returns an array of transactions to this wallet. + /// + public TransactionInfoModel[] ListTransactions(int count = 10, int skip = 0) + { + RPCResponse response = SendCommand(RPCOperations.listtransactions, "*", count, skip); + return response.Result.Select(i => new TransactionInfoModel((JObject)i)).ToArray(); + } + // listunspent /// diff --git a/src/Stratis.Bitcoin.Features.Wallet/Interfaces/IWalletManager.cs b/src/Stratis.Bitcoin.Features.Wallet/Interfaces/IWalletManager.cs index 7d8602aca8..ea4fd3c94f 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/Interfaces/IWalletManager.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/Interfaces/IWalletManager.cs @@ -70,6 +70,12 @@ public interface IWalletManager /// A collection of spendable outputs that belong to the given account. IEnumerable GetSpendableTransactionsInAccount(WalletAccountReference walletAccountReference, int confirmations = 0); + /// + /// Lists all transactions from the account specified in . + /// + /// A collection of transactions. + IEnumerable GetAllTransactionsInWallet(string walletName, Func accountFilter); + /// /// Creates a wallet and persist it as a file on the local system. /// diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletManager.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletManager.cs index 3463b1625d..e25db6943c 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletManager.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletManager.cs @@ -1137,6 +1137,21 @@ public IEnumerable GetSpendableTransactionsInAccount(Wal return res; } + public IEnumerable GetAllTransactionsInWallet(string walletName, Func accountFilter) + { + Guard.NotEmpty(walletName, nameof(walletName)); + + Wallet wallet = this.GetWallet(walletName); + IEnumerable res = null; + + lock (this.lockObject) + { + res = wallet.GetAllTransactions(accountFilter: accountFilter); + } + + return res; + } + /// public void RemoveBlocks(ChainedHeader fork) { diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletRPCController.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletRPCController.cs index 5b3301760f..cb753b73cb 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletRPCController.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletRPCController.cs @@ -933,6 +933,86 @@ public UnspentCoinModel[] ListUnspent(int minConfirmations = 1, int maxConfirmat return unspentCoins.ToArray(); } + [ActionName("listtransactions")] + [ActionDescription("Returns an array of transactions belonging to this wallet.")] + public TransactionInfoModel[] ListTransactions(string label = "*", int count = 10, int skip = 0, bool include_watchonly = true) + { + WalletAccountReference accountReference = this.GetWalletAccountReference(); + + Func accountFilter = a => + { + var labelMatch = label == "*" || a.Name == label; + + return include_watchonly + ? (a.Index == Wallet.WatchOnlyAccountIndex || a.Index < Wallet.SpecialPurposeAccountIndexesStart) && labelMatch + : (a.Index < Wallet.SpecialPurposeAccountIndexesStart) && labelMatch; + }; + + var transactions = this.walletManager.GetAllTransactionsInWallet(accountReference.WalletName, accountFilter); + + var filteredTransactions = transactions + .Skip(skip) + .Take(count) + .Select(tx => new TransactionInfoModel + { + BlockHash = tx.BlockHash.ToString(), + Amount = tx.Amount, + TxId = tx.Id.ToString(), + BlockHeight = tx.BlockHeight ?? 0, + Generated = tx.IsCoinBase ?? false, + Confirmations = tx.BlockHeight.HasValue ? (this.walletManager.LastBlockHeight() + 1) - tx.BlockHeight.Value : 0, + BlockIndex = tx.BlockIndex ?? 0 + }) + .ToArray(); + + return filteredTransactions; + } + + [ActionName("listsinceblock")] + [ActionDescription("Returns an array of transactions belonging to this wallet since specified block.")] + public TransactionsSinceBlockModel ListSinceBlock(string blockhash = "", int targetConfirmations = 1) + { + WalletAccountReference accountReference = this.GetWalletAccountReference(); + var transactions = this.walletManager.GetAllTransactionsInWallet(accountReference.WalletName, Wallet.NormalAccounts); + + int? targetBlockHeight = null; + + if (!string.IsNullOrEmpty(blockhash)) + { + var transactionWithMatchingBlock = transactions.FirstOrDefault(b => b.BlockHash.ToString().Equals(blockhash, StringComparison.OrdinalIgnoreCase)); + + if (transactionWithMatchingBlock == null) + { + return new TransactionsSinceBlockModel { Transactions = Array.Empty(), LastBlock = blockhash }; + } + + targetBlockHeight = transactionWithMatchingBlock.BlockHeight ?? 0; + } + + var filteredTransactions = transactions + .Where(tx => !targetBlockHeight.HasValue || tx.BlockHeight >= targetBlockHeight.Value) + .Select(tx => new TransactionInfoModel + { + BlockHash = tx.BlockHash.ToString(), + Amount = tx.Amount, + TxId = tx.Id.ToString(), + BlockHeight = tx.BlockHeight ?? 0, + Generated = tx.IsCoinBase ?? false, + Confirmations = tx.BlockHeight.HasValue ? (this.walletManager.LastBlockHeight() + 1) - tx.BlockHeight.Value : 0, + BlockIndex = tx.BlockIndex ?? 0 + }) + .Where(tx => tx.Confirmations >= targetConfirmations) + .ToArray(); + + return new TransactionsSinceBlockModel + { + Transactions = filteredTransactions, + LastBlock = string.IsNullOrEmpty(blockhash) + ? filteredTransactions.LastOrDefault()?.BlockHash + : blockhash + }; + } + [ActionName("sendmany")] [ActionDescription("Creates and broadcasts a transaction which sends outputs to multiple addresses.")] public async Task SendManyAsync(string fromAccount, string addressesJson, int minConf = 1, string comment = null, string subtractFeeFromJson = null, bool isReplaceable = false, int? confTarget = null, string estimateMode = "UNSET") diff --git a/src/Stratis.Bitcoin.IntegrationTests/RPC/RpcBitcoinImmutableTests.cs b/src/Stratis.Bitcoin.IntegrationTests/RPC/RpcBitcoinImmutableTests.cs index 275adcf983..6a1ff6e41a 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/RPC/RpcBitcoinImmutableTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/RPC/RpcBitcoinImmutableTests.cs @@ -3,6 +3,7 @@ using NBitcoin; using Stratis.Bitcoin.Features.RPC; using Stratis.Bitcoin.Features.RPC.Exceptions; +using Stratis.Bitcoin.Features.RPC.Models; using Stratis.Bitcoin.IntegrationTests.Common.EnvironmentMockUpHelpers; using Xunit; @@ -78,6 +79,22 @@ public async Task GetTxOutAsyncWithValidTxThenReturnsCorrectUnspentTxAsync() Assert.Equal(coin.Address.ToString(), resultTxOut.scriptPubKey.addresses[0]); } + [Fact] + public async Task GetAllTxsWithoutFiltersThenReturnsTop10TxsAsync() + { + RPCClient rpc = this.rpcTestFixture.RpcClient; + TransactionInfoModel[] transactions = rpc.ListTransactions(); + Assert.Equal(10, transactions.Length); + } + + [Fact] + public async Task GetTxOutAsyncWithValidTxThenReturnsCorrectTxsSinceBlockAsync() + { + RPCClient rpc = this.rpcTestFixture.RpcClient; + TransactionsSinceBlockModel transactions = rpc.ListSinceBlock("3ed622cc3323746716ebdc7b02f4030c945f76e84f4fd6005bef6a04353681f9"); + Assert.Equal(101, transactions.Transactions.Length); + } + /// /// NBitcoin test CanUseAsyncRPC /// From 2fdc28c62a2da12d33631fb86ea8d699dab64681 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Wed, 30 Mar 2022 21:34:55 +1100 Subject: [PATCH 02/44] Resolve various code warnings (#950) * Resolve various code warnings * Revert inadvertent commit (cherry picked from commit 8335edc13225017cb8a0a1ce42c92a11439c8b1c) --- src/NBitcoin.Tests/ChainTests.cs | 10 +++++----- .../MempoolBehavior.cs | 8 ++++++++ .../MempoolValidator.cs | 2 ++ .../PoABlockHeaderValidator.cs | 5 +++++ .../Controller/FullNodeControllerTest.cs | 2 +- src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs | 8 ++++---- .../ConsensusManagerHelper.cs | 2 +- src/Stratis.Bitcoin.Tests.Common/TransactionsHelper.cs | 3 +++ .../Receipts/IReceiptRepository.cs | 5 +++++ .../State/StateRepository.cs | 4 ++++ 10 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/NBitcoin.Tests/ChainTests.cs b/src/NBitcoin.Tests/ChainTests.cs index f729cbc794..7fe5524395 100644 --- a/src/NBitcoin.Tests/ChainTests.cs +++ b/src/NBitcoin.Tests/ChainTests.cs @@ -31,7 +31,7 @@ public void CanSaveChain() ChainedHeader fork = this.AppendBlock(chain); this.AppendBlock(chain); - var chain2 = new ChainIndexer(this.network).Load(chain.ToBytes()); + ChainIndexer chain2 = new ChainIndexer(this.network).Load(chain.ToBytes()); Assert.True(chain.Tip.HashBlock == chain2.Tip.HashBlock); } @@ -149,7 +149,7 @@ public void CanBuildChain() [Trait("UnitTest", "UnitTest")] public void CanCalculateDifficulty() { - var main = new ChainIndexer(this.network).Load(this.LoadMainChain()); + ChainIndexer main = new ChainIndexer(this.network).Load(this.LoadMainChain()); // The state of the line separators may be affected by copy operations - so do an environment independent line split... string[] histories = File.ReadAllText(TestDataLocations.GetFileFromDataFolder("targethistory.csv")).Split(new string[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries); @@ -161,7 +161,7 @@ public void CanCalculateDifficulty() BlockHeader block = main.GetHeader(height).Header; Assert.Equal(expectedTarget, block.Bits); - Target target = main.GetHeader(height).GetWorkRequired(network); + Target target = main.GetHeader(height).GetWorkRequired(this.network); Assert.Equal(expectedTarget, target); } } @@ -170,7 +170,7 @@ public void CanCalculateDifficulty() [Trait("UnitTest", "UnitTest")] public void CanValidateChain() { - var main = new ChainIndexer(this.network).Load(this.LoadMainChain()); + ChainIndexer main = new ChainIndexer(this.network).Load(this.LoadMainChain()); foreach (ChainedHeader h in main.EnumerateToTip(main.Genesis)) { Assert.True(h.Validate(this.network)); @@ -340,7 +340,7 @@ private ChainIndexer CreateChain(BlockHeader genesis, int height) { var chain = new ChainIndexer(this.network); - var chainedHeaderPrev = chain.Tip; + ChainedHeader chainedHeaderPrev = chain.Tip; for (int i = 0; i < height; i++) { diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolBehavior.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolBehavior.cs index c4290c3226..84140c3f5c 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolBehavior.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolBehavior.cs @@ -152,6 +152,7 @@ public override object Clone() /// /// TODO: Fix the exception handling of the async event. /// + /// The asynchronous task. private async Task OnMessageReceivedAsync(INetworkPeer peer, IncomingMessage message) { try @@ -176,6 +177,7 @@ private async Task OnMessageReceivedAsync(INetworkPeer peer, IncomingMessage mes /// /// Peer sending the message. /// Incoming message. + /// The asynchronous task. private async Task ProcessMessageAsync(INetworkPeer peer, IncomingMessage message) { try @@ -212,6 +214,7 @@ private async Task ProcessMessageAsync(INetworkPeer peer, IncomingMessage messag /// /// Peer sending the message. /// The message payload. + /// The asynchronous task. private async Task SendMempoolPayloadAsync(INetworkPeer peer, MempoolPayload message) { Guard.NotNull(peer, nameof(peer)); @@ -278,6 +281,7 @@ private async Task SendMempoolPayloadAsync(INetworkPeer peer, MempoolPayload mes /// /// The peer sending the message. /// The inventory payload in the message. + /// The asynchronous task. private async Task ProcessInvAsync(INetworkPeer peer, InvPayload invPayload) { Guard.NotNull(peer, nameof(peer)); @@ -338,6 +342,7 @@ private async Task ProcessInvAsync(INetworkPeer peer, InvPayload invPayload) /// /// Peer sending the message. /// The payload for the message. + /// The asynchronous task. private async Task ProcessGetDataAsync(INetworkPeer peer, GetDataPayload getDataPayload) { Guard.NotNull(peer, nameof(peer)); @@ -366,6 +371,7 @@ private async Task ProcessGetDataAsync(INetworkPeer peer, GetDataPayload getData /// /// Peer sending the message. /// The payload for the message. + /// The asynchronous task. private async Task ProcessTxPayloadAsync(INetworkPeer peer, TxPayload transactionPayload) { // Stop processing the transaction early if we are in blocks only mode. @@ -448,6 +454,7 @@ private async Task ProcessTxPayloadAsync(INetworkPeer peer, TxPayload transactio /// /// Peer to send transactions to. /// List of transactions. + /// The asynchronous task. private async Task SendAsTxInventoryAsync(INetworkPeer peer, List trxList) { var queue = new Queue(trxList.Select(s => new InventoryVector(peer.AddSupportedOptions(InventoryType.MSG_TX), s))); @@ -520,6 +527,7 @@ public void RelayTransaction(uint256 hash) /// Sends transaction inventory to attached peer. /// This is executed on a 5 second loop when MempoolSignaled is constructed. /// + /// The asynchronous task. public async Task SendTrickleAsync() { INetworkPeer peer = this.AttachedPeer; diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolValidator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolValidator.cs index f7b12d6917..7be077d659 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolValidator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolValidator.cs @@ -102,6 +102,7 @@ public class MempoolValidator : IMempoolValidator private readonly FeeRate minRelayTxFee; // TODO: Verify these + /// Flags that determine how transaction should be validated in non-consensus code. public static Transaction.LockTimeFlags StandardLocktimeVerifyFlags = Transaction.LockTimeFlags.VerifySequence | Transaction.LockTimeFlags.MedianTimePast; @@ -304,6 +305,7 @@ public static int CalculateModifiedSize(int nTxSize, Transaction trx, ConsensusO /// Validation state for creating the validation context. /// The transaction to validate. /// Not currently used + /// The asynchronous task. private async Task AcceptToMemoryPoolWorkerAsync(MempoolValidationState state, Transaction tx, List vHashTxnToUncache) { var context = new MempoolValidationContext(tx, state) diff --git a/src/Stratis.Bitcoin.Features.PoA/PoABlockHeaderValidator.cs b/src/Stratis.Bitcoin.Features.PoA/PoABlockHeaderValidator.cs index 30fc83575d..a2819d428e 100644 --- a/src/Stratis.Bitcoin.Features.PoA/PoABlockHeaderValidator.cs +++ b/src/Stratis.Bitcoin.Features.PoA/PoABlockHeaderValidator.cs @@ -15,6 +15,8 @@ public PoABlockHeaderValidator(ILoggerFactory factory) } /// Signs PoA header with the specified key. + /// Key to sign wih. + /// to sign. public void Sign(Key key, PoABlockHeader header) { uint256 headerHash = header.GetHash(); @@ -27,6 +29,9 @@ public void Sign(Key key, PoABlockHeader header) /// Verifies if signature of provided header was created using /// private key that corresponds to given public key. /// + /// of private used to sign the message. + /// to verify signature of. + /// true if verification successful or false otherwise. public bool VerifySignature(PubKey pubKey, PoABlockHeader header) { if ((header.BlockSignature == null) || header.BlockSignature.IsEmpty()) diff --git a/src/Stratis.Bitcoin.Features.RPC.Tests/Controller/FullNodeControllerTest.cs b/src/Stratis.Bitcoin.Features.RPC.Tests/Controller/FullNodeControllerTest.cs index 084cda359a..dc431d2ec7 100644 --- a/src/Stratis.Bitcoin.Features.RPC.Tests/Controller/FullNodeControllerTest.cs +++ b/src/Stratis.Bitcoin.Features.RPC.Tests/Controller/FullNodeControllerTest.cs @@ -394,7 +394,7 @@ public async Task GetTxOutAsync_IncludeInMempool_UnspentTransactionFound_VOutNot } [Fact] - public async Task GetTxOutProof_TransactionInSameSpecifiedBlock_ReturnsProof() + public async Task GetTxOutProof_TransactionInSameSpecifiedBlock_ReturnsProofAsync() { ChainedHeader block = this.chain.GetHeader(2); Transaction tx = block.Block.Transactions.First(); diff --git a/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs b/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs index 8ec9fc3483..880e71a1d0 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs @@ -188,9 +188,9 @@ private void more_blocks_mined_past_maturity_of_original_block() TestHelper.MineBlocks(this.firstStratisPowApiNode, this.maturity); } - private async Task a_real_transaction() + private void a_real_transaction() { - await this.SendTransaction(await this.BuildTransaction()); + this.SendTransactionAsync(this.BuildTransactionAsync().GetAwaiter().GetResult()).GetAwaiter().GetResult(); } private void the_block_with_the_transaction_is_mined() @@ -530,7 +530,7 @@ private void send_api_get_request(string apiendpoint) } } - private async Task SendTransaction(IActionResult transactionResult) + private async Task SendTransactionAsync(IActionResult transactionResult) { var walletTransactionModel = (WalletBuildTransactionModel)(transactionResult as JsonResult)?.Value; this.transaction = this.firstStratisPowApiNode.FullNode.Network.CreateTransaction(walletTransactionModel.Hex); @@ -538,7 +538,7 @@ await this.firstStratisPowApiNode.FullNode.NodeController() .SendTransactionAsync(new SendTransactionRequest(walletTransactionModel.Hex)); } - private async Task BuildTransaction() + private async Task BuildTransactionAsync() { IActionResult transactionResult = await this.firstStratisPowApiNode.FullNode .NodeController() diff --git a/src/Stratis.Bitcoin.Tests.Common/ConsensusManagerHelper.cs b/src/Stratis.Bitcoin.Tests.Common/ConsensusManagerHelper.cs index 0ce23564ed..5e3c8bb7f0 100644 --- a/src/Stratis.Bitcoin.Tests.Common/ConsensusManagerHelper.cs +++ b/src/Stratis.Bitcoin.Tests.Common/ConsensusManagerHelper.cs @@ -60,7 +60,7 @@ public static IServiceCollection GetMockingServices( // Dont check PoW of a header in this test. network.Consensus.ConsensusRules.HeaderValidationRules.RemoveAll(x => x == typeof(CheckDifficultyPowRule)); - var mockingServices = new ServiceCollection() + IServiceCollection mockingServices = new ServiceCollection() .AddSingleton(network) .AddSingleton(nodeSettings ?? (ctx => new NodeSettings(network))) .AddSingleton(ctx => ctx.GetService().DataFolder) diff --git a/src/Stratis.Bitcoin.Tests.Common/TransactionsHelper.cs b/src/Stratis.Bitcoin.Tests.Common/TransactionsHelper.cs index 4097f32410..881627b337 100644 --- a/src/Stratis.Bitcoin.Tests.Common/TransactionsHelper.cs +++ b/src/Stratis.Bitcoin.Tests.Common/TransactionsHelper.cs @@ -47,6 +47,9 @@ public static void CreateCirrusRewardOutput(Transaction coinstakeTransaction, Ne } /// Creates invalid PoW block with coinbase transaction. + /// The network. + /// Identifies the tip (previous block and height). + /// . public static Block CreateDummyBlockWithTransaction(Network network, ChainedHeader tip) { Block block = network.Consensus.ConsensusFactory.CreateBlock(); diff --git a/src/Stratis.SmartContracts.Core/Receipts/IReceiptRepository.cs b/src/Stratis.SmartContracts.Core/Receipts/IReceiptRepository.cs index b127443574..01ac1ac44f 100644 --- a/src/Stratis.SmartContracts.Core/Receipts/IReceiptRepository.cs +++ b/src/Stratis.SmartContracts.Core/Receipts/IReceiptRepository.cs @@ -8,17 +8,22 @@ public interface IReceiptRepository /// /// Permanently store several receipts. /// + /// Receipts to store. void Store(IEnumerable receipts); /// /// Retrieve a receipt by transaction hash. /// + /// Hash of transaction to retrieve. + /// . Receipt Retrieve(uint256 txHash); /// /// Retrieves the receipt for each of the given IDs. It will put null in an index /// if that hash is not found in the database. /// + /// Hashes for which to retrieve receipts. + /// List of receipts retrieved or null for each hash not found. IList RetrieveMany(IList hashes); } } diff --git a/src/Stratis.SmartContracts.Core/State/StateRepository.cs b/src/Stratis.SmartContracts.Core/State/StateRepository.cs index e115dd2e72..0e857017af 100644 --- a/src/Stratis.SmartContracts.Core/State/StateRepository.cs +++ b/src/Stratis.SmartContracts.Core/State/StateRepository.cs @@ -124,6 +124,8 @@ public IStateRepository StartTracking() /// /// Gets a snaphot of the state repository up until the given state root. /// + /// The state root to get a snapshot for. + /// public virtual IStateRepositoryRoot GetSnapshotTo(byte[] stateRoot) { return this.parent.GetSnapshotTo(stateRoot); @@ -159,6 +161,8 @@ public virtual void Flush() /// /// Note that because the initial transaction will always be coming from a human we don't need to minus if from. /// + /// The contract to return the balance for. + /// The balance of the contract. public ulong GetCurrentBalance(uint160 address) { ulong ret = 0; From d969b1b5dadff898092bacd158b0e45f9c23df28 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Apr 2022 15:37:05 +1000 Subject: [PATCH 03/44] [IBD/Rewind] Refactor RecoverStoreTip (#943) * Refactor RecoverStoreTip * Fix BlockStoreTests (cherry picked from commit 174a4b57c0774523c5d3fa505025cf7560eb04b0) --- .../BlockStoreQueue.cs | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs index ae245d367e..60d1ac6285 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs @@ -420,38 +420,14 @@ private ChainedHeader RecoverStoreTip() if (blockStoreTip != null) return blockStoreTip; - var blockStoreResetList = new List(); + int firstNotFound = BinarySearch.BinaryFindFirst((h) => this.chainIndexer[h] == null || this.blockRepository.GetBlock(this.chainIndexer[h].HashBlock) == null, 1, this.chainIndexer.Height); + if (firstNotFound < 0) + return this.chainIndexer.Tip; - uint256 resetBlockHash = this.blockRepository.TipHashAndHeight.Hash; - Block resetBlock = this.blockRepository.GetBlock(resetBlockHash); + ChainedHeader newTip = this.chainIndexer[firstNotFound - 1]; - while (this.chainIndexer.GetHeader(resetBlockHash) == null) - { - blockStoreResetList.Add(resetBlockHash); - - if (resetBlock.Header.HashPrevBlock == this.chainIndexer.Genesis.HashBlock) - { - resetBlockHash = this.chainIndexer.Genesis.HashBlock; - break; - } - - resetBlock = this.blockRepository.GetBlock(resetBlock.Header.HashPrevBlock); - - if (resetBlock == null) - { - // This can happen only if block store is corrupted. - throw new BlockStoreException("Block store failed to recover."); - } - - resetBlockHash = resetBlock.GetHash(); - } - - ChainedHeader newTip = this.chainIndexer.GetHeader(resetBlockHash); - - if (blockStoreResetList.Count != 0) - this.blockRepository.Delete(new HashHeightPair(newTip), blockStoreResetList); - - this.chainIndexer.Initialize(newTip); // we have to set chain store to be same as the store tip. + // Set chain store to be same as the store tip. + this.chainIndexer.Initialize(newTip); this.logger.LogWarning("Block store tip recovered to block '{0}'.", newTip); From 6b1918e0f97aeb12b29673a9364d294b2fa184cb Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Apr 2022 15:37:19 +1000 Subject: [PATCH 04/44] Add "minblksdownload" command-line option (#954) * Add "minblksdownload" command-line option * Update tests (cherry picked from commit 5a61925e0dbceeb4b12c36244d375e5eeb0a113a) --- .../BlockPulling/BlockPullerTestsHelper.cs | 10 +- .../BlockPulling/BlockPuller.cs | 108 ++++++++++++------ .../Configuration/NodeSettings.cs | 3 + 3 files changed, 83 insertions(+), 38 deletions(-) diff --git a/src/Stratis.Bitcoin.Tests/BlockPulling/BlockPullerTestsHelper.cs b/src/Stratis.Bitcoin.Tests/BlockPulling/BlockPullerTestsHelper.cs index d1695052df..3beda6a6f9 100644 --- a/src/Stratis.Bitcoin.Tests/BlockPulling/BlockPullerTestsHelper.cs +++ b/src/Stratis.Bitcoin.Tests/BlockPulling/BlockPullerTestsHelper.cs @@ -152,13 +152,15 @@ public ExtendedBlockPuller(IChainState chainState, NodeSettings nodeSettings, ID public Dictionary> AssignedHeadersByPeerId => (Dictionary>)this.puller.GetMemberValue("assignedHeadersByPeerId"); - public int PeerSpeedLimitWhenNotInIbdBytesPerSec => typeof(BlockPuller).GetPrivateConstantValue("PeerSpeedLimitWhenNotInIbdBytesPerSec"); + public int PeerSpeedLimitWhenNotInIbdBytesPerSec => this.BlockPullerSettings.PeerSpeedLimitWhenNotInIbdBytesPerSec; - public int ImportantHeightMargin => typeof(BlockPuller).GetPrivateConstantValue("ImportantHeightMargin"); + private BlockPuller.Settings BlockPullerSettings => (BlockPuller.Settings)this.puller.GetMemberValue("settings"); - public int StallingLoopIntervalMs => typeof(BlockPuller).GetPrivateConstantValue("StallingLoopIntervalMs"); + public int ImportantHeightMargin => this.BlockPullerSettings.ImportantHeightMargin; - public int MaxSecondsToDeliverBlock => typeof(BlockPuller).GetPrivateConstantValue("MaxSecondsToDeliverBlock"); + public int StallingLoopIntervalMs => this.BlockPullerSettings.StallingLoopIntervalMs; + + public int MaxSecondsToDeliverBlock => this.BlockPullerSettings.MaxSecondsToDeliverBlock; public void RecalculateQualityScoreLocked(IBlockPullerBehavior pullerBehavior, int peerId) { diff --git a/src/Stratis.Bitcoin/BlockPulling/BlockPuller.cs b/src/Stratis.Bitcoin/BlockPulling/BlockPuller.cs index 790e290ee5..7df007ef71 100644 --- a/src/Stratis.Bitcoin/BlockPulling/BlockPuller.cs +++ b/src/Stratis.Bitcoin/BlockPulling/BlockPuller.cs @@ -85,24 +85,69 @@ public interface IBlockPuller : IDisposable public class BlockPuller : IBlockPuller { - /// Interval between checking if peers that were assigned important blocks didn't deliver the block. - private const int StallingLoopIntervalMs = 500; + public class Settings + { + /// Amount of samples that should be used for average block size calculation. + public int AverageBlockSizeSamplesCount => 1000; - /// The minimum empty slots percentage to start processing . - private const double MinEmptySlotsPercentageToStartProcessingTheQueue = 0.1; + /// The minimal count of blocks that we can ask for simultaneous download. + public int MinimalCountOfBlocksBeingDownloaded { get; private set; } = 10; - /// - /// Defines which blocks are considered to be important. - /// If requested block height is less than out consensus tip height plus this value then the block is considered to be important. - /// - private const int ImportantHeightMargin = 10; + /// The maximum blocks being downloaded multiplier. Value of 1.1 means that we will ask for 10% more than we estimated peers can deliver. + public double MaxBlocksBeingDownloadedMultiplier => 1.1; + + /// Interval between checking if peers that were assigned important blocks didn't deliver the block. + public int StallingLoopIntervalMs => 500; + + /// The minimum empty slots percentage to start processing . + public double MinEmptySlotsPercentageToStartProcessingTheQueue => 0.1; + + /// + /// Defines which blocks are considered to be important. + /// If requested block height is less than out consensus tip height plus this value then the block is considered to be important. + /// + public int ImportantHeightMargin => 10; - /// The maximum time in seconds in which peer should deliver an assigned block. - /// If peer fails to deliver in that time his assignments will be released and the peer penalized. - private const int MaxSecondsToDeliverBlock = 30; // TODO change to target spacing / 3 + /// The maximum time in seconds in which peer should deliver an assigned block. + /// If peer fails to deliver in that time his assignments will be released and the peer penalized. + public int MaxSecondsToDeliverBlock => 30; // TODO change to target spacing / 3 - /// This affects quality score only. If the peer is too fast don't give him all the assignments in the world when not in IBD. - private const int PeerSpeedLimitWhenNotInIbdBytesPerSec = 1024 * 1024; + /// This affects quality score only. If the peer is too fast don't give him all the assignments in the world when not in IBD. + public int PeerSpeedLimitWhenNotInIbdBytesPerSec => 1024 * 1024; + + public Settings(NodeSettings nodeSettings) + { + this.MinimalCountOfBlocksBeingDownloaded = nodeSettings.ConfigReader.GetOrDefault("minblksdownload", this.MinimalCountOfBlocksBeingDownloaded); + } + + /// + /// Get the default configuration. + /// + /// The string builder to add the settings to. + /// The network to base the defaults off. + public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) + { + builder.AppendLine("####BlockPuller Settings####"); + builder.AppendLine($"#The minimum number of blocks to download. Default 10."); + builder.AppendLine($"#minblksdownload=10"); + } + + /// + /// Displays command-line help. + /// + /// The network to extract values from. + public static void PrintHelp(Network network) + { + Guard.NotNull(network, nameof(network)); + + var defaults = NodeSettings.Default(network: network); + + var builder = new StringBuilder(); + builder.AppendLine($"-minblksdownload= Minimum number of blocks to download. Defaults to 10."); + + defaults.Logger.LogInformation(builder.ToString()); + } + } /// Hash of the delivered block. /// The block. @@ -144,15 +189,6 @@ public class BlockPuller : IBlockPuller /// Write access to this object has to be protected by . private readonly AverageCalculator averageBlockSizeBytes; - /// Amount of samples that should be used for average block size calculation. - private const int AverageBlockSizeSamplesCount = 1000; - - /// The minimal count of blocks that we can ask for simultaneous download. - private const int MinimalCountOfBlocksBeingDownloaded = 10; - - /// The maximum blocks being downloaded multiplier. Value of 1.1 means that we will ask for 10% more than we estimated peers can deliver. - private const double MaxBlocksBeingDownloadedMultiplier = 1.1; - /// Signaler that triggers and processing when set. /// This object has to be protected by . private readonly AsyncManualResetEvent processQueuesSignal; @@ -192,6 +228,9 @@ public class BlockPuller : IBlockPuller /// private readonly IChainState chainState; + /// + private readonly Settings settings; + /// /// This object has to be protected by . private readonly NetworkPeerRequirement networkPeerRequirement; @@ -219,7 +258,8 @@ public BlockPuller(IChainState chainState, NodeSettings nodeSettings, IDateTimeP this.assignedDownloadsSorted = new LinkedList(); this.assignedHeadersByPeerId = new Dictionary>(); - this.averageBlockSizeBytes = new AverageCalculator(AverageBlockSizeSamplesCount); + this.settings = new Settings(nodeSettings); + this.averageBlockSizeBytes = new AverageCalculator(this.settings.AverageBlockSizeSamplesCount); this.pullerBehaviorsByPeerId = new Dictionary(); @@ -238,7 +278,7 @@ public BlockPuller(IChainState chainState, NodeSettings nodeSettings, IDateTimeP this.cancellationSource = new CancellationTokenSource(); this.random = new Random(); - this.maxBlocksBeingDownloaded = MinimalCountOfBlocksBeingDownloaded; + this.maxBlocksBeingDownloaded = this.settings.MinimalCountOfBlocksBeingDownloaded; this.chainState = chainState; this.dateTimeProvider = dateTimeProvider; @@ -406,7 +446,7 @@ private async Task StallingLoopAsync() { try { - await Task.Delay(StallingLoopIntervalMs, this.cancellationSource.Token).ConfigureAwait(false); + await Task.Delay(this.settings.StallingLoopIntervalMs, this.cancellationSource.Token).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -436,7 +476,7 @@ private async Task AssignDownloadJobsAsync() emptySlots = this.maxBlocksBeingDownloaded - this.assignedDownloadsByHash.Count; } - int slotsThreshold = (int)(this.maxBlocksBeingDownloaded * MinEmptySlotsPercentageToStartProcessingTheQueue); + int slotsThreshold = (int)(this.maxBlocksBeingDownloaded * this.settings.MinEmptySlotsPercentageToStartProcessingTheQueue); if (emptySlots >= slotsThreshold) this.ProcessQueueLocked(this.downloadJobsQueue, newAssignments, failedHashes, emptySlots); @@ -720,7 +760,7 @@ private List DistributeHeadersLocked(DownloadJob downloadJob, /// Checks if peers failed to deliver important blocks and penalizes them if they did. private void CheckStalling() { - int lastImportantHeight = this.chainState.ConsensusTip.Height + ImportantHeightMargin; + int lastImportantHeight = this.chainState.ConsensusTip.Height + this.settings.ImportantHeightMargin; this.logger.LogDebug("Blocks up to height {0} are considered to be important.", lastImportantHeight); var allReleasedAssignments = new List>>(); @@ -744,7 +784,7 @@ private void CheckStalling() int peerId = current.Value.PeerId; current = current.Next; - if (secondsPassed < MaxSecondsToDeliverBlock) + if (secondsPassed < this.settings.MaxSecondsToDeliverBlock) continue; // Peer already added to the collection of peers to release and reassign. @@ -849,8 +889,8 @@ private void RecalculateQualityScoreLocked(IBlockPullerBehavior pullerBehavior, long bestSpeed = this.pullerBehaviorsByPeerId.Max(x => x.Value.SpeedBytesPerSecond); long adjustedBestSpeed = bestSpeed; - if (!this.isIbd && (adjustedBestSpeed > PeerSpeedLimitWhenNotInIbdBytesPerSec)) - adjustedBestSpeed = PeerSpeedLimitWhenNotInIbdBytesPerSec; + if (!this.isIbd && (adjustedBestSpeed > this.settings.PeerSpeedLimitWhenNotInIbdBytesPerSec)) + adjustedBestSpeed = this.settings.PeerSpeedLimitWhenNotInIbdBytesPerSec; if (pullerBehavior.SpeedBytesPerSecond != bestSpeed) { @@ -876,10 +916,10 @@ private void RecalculateMaxBlocksBeingDownloadedLocked() { // How many blocks we can download in 1 second. if (this.averageBlockSizeBytes.Average > 0) - this.maxBlocksBeingDownloaded = (int)((this.GetTotalSpeedOfAllPeersBytesPerSec() * MaxBlocksBeingDownloadedMultiplier) / this.averageBlockSizeBytes.Average); + this.maxBlocksBeingDownloaded = (int)((this.GetTotalSpeedOfAllPeersBytesPerSec() * this.settings.MaxBlocksBeingDownloadedMultiplier) / this.averageBlockSizeBytes.Average); - if (this.maxBlocksBeingDownloaded < MinimalCountOfBlocksBeingDownloaded) - this.maxBlocksBeingDownloaded = MinimalCountOfBlocksBeingDownloaded; + if (this.maxBlocksBeingDownloaded < this.settings.MinimalCountOfBlocksBeingDownloaded) + this.maxBlocksBeingDownloaded = this.settings.MinimalCountOfBlocksBeingDownloaded; this.logger.LogDebug("Max number of blocks that can be downloaded at the same time is set to {0}.", this.maxBlocksBeingDownloaded); } diff --git a/src/Stratis.Bitcoin/Configuration/NodeSettings.cs b/src/Stratis.Bitcoin/Configuration/NodeSettings.cs index 9c140c756e..c0d1990e0b 100644 --- a/src/Stratis.Bitcoin/Configuration/NodeSettings.cs +++ b/src/Stratis.Bitcoin/Configuration/NodeSettings.cs @@ -8,6 +8,7 @@ using NBitcoin; using NBitcoin.Networks; using NBitcoin.Protocol; +using Stratis.Bitcoin.BlockPulling; using Stratis.Bitcoin.Builder.Feature; using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Configuration.Settings; @@ -429,6 +430,7 @@ public static void PrintHelp(Network network) defaults.Logger.LogInformation(builder.ToString()); + BlockPuller.Settings.PrintHelp(network); ConnectionManagerSettings.PrintHelp(network); } @@ -452,6 +454,7 @@ public static void BuildDefaultConfigurationFile(StringBuilder builder, Network builder.AppendLine($"#minrelaytxfee={network.MinRelayTxFee}"); builder.AppendLine(); + BlockPuller.Settings.BuildDefaultConfigurationFile(builder, network); ConnectionManagerSettings.BuildDefaultConfigurationFile(builder, network); } From 81d6c47d863102d6d8ca2044345555bb27262b39 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Apr 2022 15:38:45 +1000 Subject: [PATCH 05/44] [IBD/Rewind] Forego unused transactions and unnecessary commits in polls repo (#936) * Forego unused transactions and unnecessary commits in polls repo * Initialize CurrentTip at genesis --- .../PollsRepositoryTests.cs | 50 ++- .../Voting/PollsRepository.cs | 370 ++++++++++-------- .../Voting/VotingManager.cs | 94 ++--- 3 files changed, 268 insertions(+), 246 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.PoA.Tests/PollsRepositoryTests.cs b/src/Stratis.Bitcoin.Features.PoA.Tests/PollsRepositoryTests.cs index cf0d9bb3f2..1bf017d997 100644 --- a/src/Stratis.Bitcoin.Features.PoA.Tests/PollsRepositoryTests.cs +++ b/src/Stratis.Bitcoin.Features.PoA.Tests/PollsRepositoryTests.cs @@ -31,11 +31,11 @@ public void CantAddOrRemovePollsOutOfOrder() this.repository.WithTransaction(transaction => { - this.repository.AddPolls(transaction, new Poll() { Id = 0 }); - this.repository.AddPolls(transaction, new Poll() { Id = 1 }); - this.repository.AddPolls(transaction, new Poll() { Id = 2 }); - Assert.Throws(() => this.repository.AddPolls(transaction, new Poll() { Id = 5 })); - this.repository.AddPolls(transaction, new Poll() { Id = 3 }); + transaction.AddPolls(new Poll() { Id = 0 }); + transaction.AddPolls(new Poll() { Id = 1 }); + transaction.AddPolls(new Poll() { Id = 2 }); + Assert.Throws(() => transaction.AddPolls(new Poll() { Id = 5 })); + transaction.AddPolls(new Poll() { Id = 3 }); transaction.Commit(); }); @@ -44,14 +44,14 @@ public void CantAddOrRemovePollsOutOfOrder() this.repository.WithTransaction(transaction => { - this.repository.RemovePolls(transaction, 3); + transaction.RemovePolls(3); - Assert.Throws(() => this.repository.RemovePolls(transaction, 6)); - Assert.Throws(() => this.repository.RemovePolls(transaction, 3)); + Assert.Throws(() => transaction.RemovePolls(6)); + Assert.Throws(() => transaction.RemovePolls(3)); - this.repository.RemovePolls(transaction, 2); - this.repository.RemovePolls(transaction, 1); - this.repository.RemovePolls(transaction, 0); + transaction.RemovePolls(2); + transaction.RemovePolls(1); + transaction.RemovePolls(0); transaction.Commit(); }); @@ -64,11 +64,9 @@ public void SavesHighestPollId() { this.repository.WithTransaction(transaction => { - this.repository.AddPolls(transaction, new Poll() { Id = 0, PollStartBlockData = new HashHeightPair(1, 1) }); - this.repository.AddPolls(transaction, new Poll() { Id = 1, PollStartBlockData = new HashHeightPair(2, 2) }); - this.repository.AddPolls(transaction, new Poll() { Id = 2, PollStartBlockData = new HashHeightPair(3, 3) }); - - this.repository.SaveCurrentTip(transaction, new HashHeightPair(this.chainIndexer.Tip.HashBlock, 0)); + transaction.AddPolls(new Poll() { Id = 0, PollStartBlockData = new HashHeightPair(1, 1) }); + transaction.AddPolls(new Poll() { Id = 1, PollStartBlockData = new HashHeightPair(2, 2) }); + transaction.AddPolls(new Poll() { Id = 2, PollStartBlockData = new HashHeightPair(3, 3) }); transaction.Commit(); }); @@ -83,19 +81,19 @@ public void CanLoadPolls() { this.repository.WithTransaction(transaction => { - this.repository.AddPolls(transaction, new Poll() { Id = 0 }); - this.repository.AddPolls(transaction, new Poll() { Id = 1 }); - this.repository.AddPolls(transaction, new Poll() { Id = 2 }); + transaction.AddPolls(new Poll() { Id = 0 }); + transaction.AddPolls(new Poll() { Id = 1 }); + transaction.AddPolls(new Poll() { Id = 2 }); transaction.Commit(); }); this.repository.WithTransaction(transaction => { - Assert.True(this.repository.GetPolls(transaction, 0, 1, 2).Count == 3); - Assert.True(this.repository.GetAllPolls(transaction).Count == 3); - Assert.Throws(() => this.repository.GetPolls(transaction, -1)); - Assert.Throws(() => this.repository.GetPolls(transaction, 9)); + Assert.True(transaction.GetPolls( 0, 1, 2).Count == 3); + Assert.True(transaction.GetAllPolls().Count == 3); + Assert.Throws(() => transaction.GetPolls(-1)); + Assert.Throws(() => transaction.GetPolls(9)); }); } @@ -106,17 +104,17 @@ public void CanUpdatePolls() this.repository.WithTransaction(transaction => { - this.repository.AddPolls(transaction, poll); + transaction.AddPolls(poll); poll.VotingData.Key = VoteKey.KickFederationMember; - this.repository.UpdatePoll(transaction, poll); + transaction.UpdatePoll(poll); transaction.Commit(); }); this.repository.WithTransaction(transaction => { - Assert.Equal(VoteKey.KickFederationMember, this.repository.GetPolls(transaction, poll.Id).First().VotingData.Key); + Assert.Equal(VoteKey.KickFederationMember, transaction.GetPolls(poll.Id).First().VotingData.Key); }); } } diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/PollsRepository.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/PollsRepository.cs index 129896e406..750d621bd0 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/PollsRepository.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/PollsRepository.cs @@ -35,6 +35,196 @@ public class PollsRepository : IDisposable private readonly PoANetwork network; + public class Transaction : IDisposable + { + private readonly PollsRepository pollsRepository; + public bool IsModified { get; private set; } + + private DBreeze.Transactions.Transaction _transaction; + private DBreeze.Transactions.Transaction transaction + { + get + { + if (this._transaction == null) + this._transaction = this.pollsRepository.dbreeze.GetTransaction(); + + return this._transaction; + } + + set + { + this._transaction = value; + } + } + + public Transaction(PollsRepository pollsRepository) + { + this.pollsRepository = pollsRepository; + this.IsModified = false; + } + + public void Insert(string tableName, TKey key, TValue value) + { + this.transaction.Insert(tableName, key, value); + this.IsModified = true; + } + + public Row Select(string tableName, TKey key) + { + return this.transaction.Select(tableName, key); + } + + public Dictionary SelectDictionary(string tableName) + { + return this.transaction.SelectDictionary(tableName); + } + + public void RemoveKey(string tableName, TKey key) + { + this.transaction.RemoveKey(tableName, key); + this.IsModified = true; + } + + public void RemoveAllKeys(string tableName, bool withFileRecreation) + { + this.transaction.RemoveAllKeys(tableName, withFileRecreation); + this.IsModified = true; + } + + public void Flush() + { + this.Commit(); + + this.transaction.Dispose(); + this.transaction = null; + } + + public void SetTip(ChainedHeader tip) + { + this.pollsRepository.CurrentTip = new HashHeightPair(tip); + this.IsModified = true; + } + + public void Commit() + { + if (this.IsModified) + { + this.SaveCurrentTip(); + this.transaction.Commit(); + this.IsModified = false; + } + } + + public void Dispose() + { + this.transaction?.Dispose(); + } + + /// Adds new polls. + /// The polls to add. + public void AddPolls(params Poll[] polls) + { + foreach (Poll pollToAdd in polls.OrderBy(p => p.Id)) + { + if (pollToAdd.Id != this.pollsRepository.highestPollId + 1) + throw new ArgumentException("Id is incorrect. Gaps are not allowed."); + + byte[] bytes = this.pollsRepository.dBreezeSerializer.Serialize(pollToAdd); + + this.Insert(DataTable, pollToAdd.Id.ToBytes(), bytes); + + this.pollsRepository.highestPollId++; + } + } + + /// Updates existing poll. + /// The poll to update. + public void UpdatePoll(Poll poll) + { + byte[] bytes = this.pollsRepository.dBreezeSerializer.Serialize(poll); + + this.Insert(DataTable, poll.Id.ToBytes(), bytes); + + } + + /// Loads polls under provided keys from the database. + /// The ids of the polls to retrieve. + /// A list of retrieved entries. + public List GetPolls(params int[] ids) + { + var polls = new List(ids.Length); + + foreach (int id in ids) + { + Row row = this.Select(DataTable, id.ToBytes()); + + if (!row.Exists) + throw new ArgumentException("Value under provided key doesn't exist!"); + + Poll poll = this.pollsRepository.dBreezeSerializer.Deserialize(row.Value); + + polls.Add(poll); + } + + return polls; + } + + /// Loads all polls from the database. + /// A list of retrieved entries. + public List GetAllPolls() + { + Dictionary data = this.SelectDictionary(DataTable); + + return data + .Where(d => d.Key.Length == 4) + .Select(d => this.pollsRepository.dBreezeSerializer.Deserialize(d.Value)) + .ToList(); + } + + private void SaveCurrentTip() + { + if (this.pollsRepository.CurrentTip != null) + { + this.Insert(DataTable, RepositoryTipKey, this.pollsRepository.dBreezeSerializer.Serialize(this.pollsRepository.CurrentTip)); + } + } + + /// Removes polls for the provided ids. + /// The ids of the polls to remove. + public void DeletePollsAndSetHighestPollId(params int[] ids) + { + foreach (int pollId in ids.OrderBy(a => a)) + { + this.RemoveKey(DataTable, pollId.ToBytes()); + } + + List polls = this.GetAllPolls(); + this.pollsRepository.highestPollId = (polls.Count == 0) ? -1 : polls.Max(a => a.Id); + } + + /// Removes polls under provided ids. + /// The ids of the polls to remove. + public void RemovePolls(params int[] ids) + { + foreach (int pollId in ids.OrderBy(id => id).Reverse()) + { + if (this.pollsRepository.highestPollId != pollId) + throw new ArgumentException("Only deletion of the most recent item is allowed!"); + + this.RemoveKey(DataTable, pollId.ToBytes()); + + this.pollsRepository.highestPollId--; + } + } + + public void ResetLocked() + { + this.pollsRepository.highestPollId = -1; + this.RemoveAllKeys(DataTable, true); + this.pollsRepository.CurrentTip = new HashHeightPair(this.pollsRepository.network.GenesisHash, 0); + } + } + public PollsRepository(ChainIndexer chainIndexer, DataFolder dataFolder, DBreezeSerializer dBreezeSerializer, PoANetwork network) { Guard.NotEmpty(dataFolder.PollsPath, nameof(dataFolder.PollsPath)); @@ -44,6 +234,7 @@ public PollsRepository(ChainIndexer chainIndexer, DataFolder dataFolder, DBreeze this.dbreeze = new DBreezeEngine(dataFolder.PollsPath); this.dBreezeSerializer = dBreezeSerializer; this.network = network; + this.CurrentTip = new HashHeightPair(network.GenesisHash, 0); this.logger = LogManager.GetCurrentClassLogger(); } @@ -53,11 +244,14 @@ public void Initialize() // Load highest index. lock (this.lockObject) { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + this.highestPollId = -1; + this.CurrentTip = new HashHeightPair(this.network.GenesisHash, 0); + + using (var transaction = new Transaction(this)) { try { - List polls = GetAllPolls(transaction); + List polls = transaction.GetAllPolls(); // If the polls repository contains duplicate polls then reset the highest poll id and // set the tip to null. @@ -68,7 +262,7 @@ public void Initialize() { this.logger.LogWarning("The polls repository contains {0} duplicate polls, it will be rebuilt.", polls.Count - uniquePolls.Count); - this.ResetLocked(transaction); + transaction.ResetLocked(); transaction.Commit(); return; } @@ -78,7 +272,8 @@ public void Initialize() if (!rowTip.Exists) { this.logger.LogInformation("The polls repository tip is unknown, it will be rebuilt."); - this.ResetLocked(transaction); + + transaction.ResetLocked(); transaction.Commit(); return; } @@ -113,7 +308,8 @@ public void Initialize() if (maxGoodHeight == -1) { this.logger.LogInformation("No common blocks found; the repo will be rebuilt from scratch."); - this.ResetLocked(transaction); + + transaction.ResetLocked(); transaction.Commit(); return; } @@ -164,11 +360,10 @@ public void Initialize() } if (modified) - UpdatePoll(transaction, poll); + transaction.UpdatePoll(poll); } - DeletePollsAndSetHighestPollId(transaction, pollsToDelete.Select(p => p.Id).ToArray()); - SaveCurrentTip(transaction, this.CurrentTip); + transaction.DeletePollsAndSetHighestPollId(pollsToDelete.Select(p => p.Id).ToArray()); transaction.Commit(); this.logger.LogInformation("Polls repository initialized at height {0}; highest poll id: {1}.", this.CurrentTip.Height, this.highestPollId); @@ -176,49 +371,26 @@ public void Initialize() catch (Exception err) when (err.Message == "No more byte to read") { this.logger.LogWarning("There was an error reading the polls repository, it will be rebuild."); - this.ResetLocked(transaction); + + transaction.ResetLocked(); transaction.Commit(); } } } } - private void ResetLocked(DBreeze.Transactions.Transaction transaction) - { - this.highestPollId = -1; - transaction.RemoveAllKeys(DataTable, true); - this.CurrentTip = null; - } - public void Reset() { lock (this.lockObject) { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + using (var transaction = new Transaction(this)) { - ResetLocked(transaction); + transaction.ResetLocked(); transaction.Commit(); } } } - public void SaveCurrentTip(DBreeze.Transactions.Transaction transaction, ChainedHeader tip) - { - SaveCurrentTip(transaction, (tip == null) ? null : new HashHeightPair(tip)); - } - - public void SaveCurrentTip(DBreeze.Transactions.Transaction transaction, HashHeightPair tip = null) - { - lock (this.lockObject) - { - if (tip != null) - this.CurrentTip = tip; - - if (transaction != null) - transaction.Insert(DataTable, RepositoryTipKey, this.dBreezeSerializer.Serialize(this.CurrentTip)); - } - } - /// Provides Id of the most recently added poll. /// Id of the most recently added poll. public int GetHighestPollId() @@ -234,148 +406,28 @@ public void Synchronous(Action action) } } - /// Removes polls for the provided ids. - /// See . - /// The ids of the polls to remove. - public void DeletePollsAndSetHighestPollId(DBreeze.Transactions.Transaction transaction, params int[] ids) + public T WithTransaction(Func func) { lock (this.lockObject) { - foreach (int pollId in ids.OrderBy(a => a)) - { - transaction.RemoveKey(DataTable, pollId.ToBytes()); - } - - List polls = GetAllPolls(transaction); - this.highestPollId = (polls.Count == 0) ? -1 : polls.Max(a => a.Id); - } - } - - /// Removes polls under provided ids. - /// See . - /// The ids of the polls to remove. - public void RemovePolls(DBreeze.Transactions.Transaction transaction, params int[] ids) - { - lock (this.lockObject) - { - foreach (int pollId in ids.OrderBy(id => id).Reverse()) - { - if (this.highestPollId != pollId) - throw new ArgumentException("Only deletion of the most recent item is allowed!"); - - transaction.RemoveKey(DataTable, pollId.ToBytes()); - - this.highestPollId--; - } - } - } - - public DBreeze.Transactions.Transaction GetTransaction() - { - lock (this.lockObject) - { - return this.dbreeze.GetTransaction(); - } - } - - public T WithTransaction(Func func) - { - lock (this.lockObject) - { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + using (var transaction = new Transaction(this)) { return func(transaction); } } } - public void WithTransaction(Action action) + public void WithTransaction(Action action) { lock (this.lockObject) { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + using (var transaction = new Transaction(this)) { action(transaction); } } } - /// Adds new polls. - /// See . - /// The polls to add. - public void AddPolls(DBreeze.Transactions.Transaction transaction, params Poll[] polls) - { - lock (this.lockObject) - { - foreach (Poll pollToAdd in polls.OrderBy(p => p.Id)) - { - if (pollToAdd.Id != this.highestPollId + 1) - throw new ArgumentException("Id is incorrect. Gaps are not allowed."); - - byte[] bytes = this.dBreezeSerializer.Serialize(pollToAdd); - - transaction.Insert(DataTable, pollToAdd.Id.ToBytes(), bytes); - - this.highestPollId++; - } - } - } - - /// Updates existing poll. - /// See . - /// The poll to update. - public void UpdatePoll(DBreeze.Transactions.Transaction transaction, Poll poll) - { - lock (this.lockObject) - { - byte[] bytes = this.dBreezeSerializer.Serialize(poll); - - transaction.Insert(DataTable, poll.Id.ToBytes(), bytes); - } - } - - /// Loads polls under provided keys from the database. - /// See . - /// The ids of the polls to retrieve. - /// A list of retrieved entries. - public List GetPolls(DBreeze.Transactions.Transaction transaction, params int[] ids) - { - lock (this.lockObject) - { - var polls = new List(ids.Length); - - foreach (int id in ids) - { - Row row = transaction.Select(DataTable, id.ToBytes()); - - if (!row.Exists) - throw new ArgumentException("Value under provided key doesn't exist!"); - - Poll poll = this.dBreezeSerializer.Deserialize(row.Value); - - polls.Add(poll); - } - - return polls; - } - } - - /// Loads all polls from the database. - /// See . - /// A list of retrieved entries. - public List GetAllPolls(DBreeze.Transactions.Transaction transaction) - { - lock (this.lockObject) - { - Dictionary data = transaction.SelectDictionary(DataTable); - - return data - .Where(d => d.Key.Length == 4) - .Select(d => this.dBreezeSerializer.Deserialize(d.Value)) - .ToList(); - } - } - private static int GetPollExpiryHeight(Poll poll, PoANetwork network) { return Math.Max(poll.PollStartBlockData.Height + network.ConsensusOptions.PollExpiryBlocks, network.ConsensusOptions.Release1100ActivationHeight); diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs index d0e1b9105b..9f99c71afc 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs @@ -109,7 +109,7 @@ public void Initialize(IFederationHistory federationHistory, IIdleFederationMemb this.idleFederationMembersKicker = idleFederationMembersKicker; this.PollsRepository.Initialize(); - this.PollsRepository.WithTransaction(transaction => this.polls = new PollsCollection(this.network as PoANetwork, this.PollsRepository.GetAllPolls(transaction))); + this.PollsRepository.WithTransaction(transaction => this.polls = new PollsCollection(this.network as PoANetwork, transaction.GetAllPolls())); this.blockConnectedSubscription = this.signals.Subscribe(this.OnBlockConnected); this.blockDisconnectedSubscription = this.signals.Subscribe(this.OnBlockDisconnected); @@ -322,7 +322,7 @@ public bool AlreadyVotingFor(VoteKey voteKey, byte[] federationMemberBytes, bool return false; } - public Poll CreatePendingPoll(DBreeze.Transactions.Transaction transaction, VotingData votingData, ChainedHeader chainedHeader, List pubKeysVotedInFavor = null) + public Poll CreatePendingPoll(PollsRepository.Transaction transaction, VotingData votingData, ChainedHeader chainedHeader, List pubKeysVotedInFavor = null) { Poll poll = null; @@ -341,9 +341,10 @@ public Poll CreatePendingPoll(DBreeze.Transactions.Transaction transaction, Voti this.polls.Add(poll); - this.PollsRepository.AddPolls(transaction, poll); + transaction.AddPolls(poll); - this.logger.LogInformation("New poll was created: '{0}'.", poll); + this.logger.LogInformation("Created poll {0} [{1}] at height {2}.", + poll.Id, this.pollResultExecutor.ConvertToString(poll.VotingData), poll.PollStartBlockData.Height); }); return poll; @@ -517,7 +518,7 @@ private bool IsVotingOnMultisigMember(VotingData votingData) return this.federationManager.IsMultisigMember(member.PubKey); } - private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedHeaderBlock chBlock) + private void ProcessBlock(PollsRepository.Transaction transaction, ChainedHeaderBlock chBlock) { long flagFall = DateTime.Now.Ticks; @@ -526,9 +527,6 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH lock (this.locker) { this.signals.Publish(new VotingManagerProcessBlock(chBlock, transaction)); - - bool pollsRepositoryModified = false; - foreach (Poll poll in this.polls.GetPollsToExecuteOrExpire(chBlock.ChainedHeader.Height)) { if (!poll.IsApproved) @@ -538,8 +536,7 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH // Flag the poll as expired. The "PollVotedInFavorBlockData" will always be null at this point due to the "GetPendingPolls" filter above. // The value of the hash is not significant but we set it to a non-zero value to prevent the field from being de-serialized as null. this.polls.AdjustPoll(poll, poll => poll.IsExpired = true); - this.PollsRepository.UpdatePoll(transaction, poll); - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); } else { @@ -547,9 +544,7 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH this.pollResultExecutor.ApplyChange(poll.VotingData); this.polls.AdjustPoll(poll, poll => poll.PollExecutedBlockData = new HashHeightPair(chBlock.ChainedHeader)); - this.PollsRepository.UpdatePoll(transaction, poll); - - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); } } @@ -560,7 +555,7 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH if (rawVotingData == null) { - this.PollsRepository.SaveCurrentTip(pollsRepositoryModified ? transaction : null, chBlock.ChainedHeader); + transaction.SetTip(chBlock.ChainedHeader); this.logger.LogTrace($"'{chBlock.ChainedHeader}' does not contain any voting data."); return; } @@ -571,7 +566,7 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH this.logger.LogError("The block was mined by a non-federation-member!"); this.logger.LogTrace("(-)[ALIEN_BLOCK]"); - this.PollsRepository.SaveCurrentTip(pollsRepositoryModified ? transaction : null, chBlock.ChainedHeader); + transaction.SetTip(chBlock.ChainedHeader); return; } @@ -611,13 +606,11 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH } poll = CreatePendingPoll(transaction, data, chBlock.ChainedHeader, new List() { new Vote() { PubKey = fedMemberKeyHex, Height = chBlock.ChainedHeader.Height } }); - pollsRepositoryModified = true; } else if (!poll.PubKeysHexVotedInFavor.Any(v => v.PubKey == fedMemberKeyHex)) { poll.PubKeysHexVotedInFavor.Add(new Vote() { PubKey = fedMemberKeyHex, Height = chBlock.ChainedHeader.Height }); - this.PollsRepository.UpdatePoll(transaction, poll); - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); this.logger.LogDebug("Voted on existing poll: '{0}'.", poll); } @@ -668,12 +661,11 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH continue; this.polls.AdjustPoll(poll, poll => poll.PollVotedInFavorBlockData = new HashHeightPair(chBlock.ChainedHeader)); - this.PollsRepository.UpdatePoll(transaction, poll); - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); } } - this.PollsRepository.SaveCurrentTip(pollsRepositoryModified ? transaction : null, chBlock.ChainedHeader); + transaction.SetTip(chBlock.ChainedHeader); } } catch (Exception ex) @@ -690,10 +682,8 @@ private void ProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedH } } - private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, ChainedHeaderBlock chBlock) + private void UnProcessBlock(PollsRepository.Transaction transaction, ChainedHeaderBlock chBlock) { - bool pollsRepositoryModified = false; - lock (this.locker) { foreach (Poll poll in this.polls.Where(x => !x.IsPending && x.PollExecutedBlockData?.Hash == chBlock.ChainedHeader.HashBlock).ToList()) @@ -702,8 +692,7 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine this.pollResultExecutor.RevertChange(poll.VotingData); this.polls.AdjustPoll(poll, poll => poll.PollExecutedBlockData = null); - this.PollsRepository.UpdatePoll(transaction, poll); - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); } foreach (Poll poll in this.polls.Where(x => x.IsExpired && !PollsRepository.IsPollExpiredAt(x, chBlock.ChainedHeader.Height - 1, this.network as PoANetwork)).ToList()) @@ -712,8 +701,7 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine // Revert back to null as this field would have been when the poll was expired. this.polls.AdjustPoll(poll, poll => poll.IsExpired = false); - this.PollsRepository.UpdatePoll(transaction, poll); - pollsRepositoryModified = true; + transaction.UpdatePoll(poll); } if (this.federationManager.GetMultisigMinersApplicabilityHeight() == chBlock.ChainedHeader.Height) @@ -726,7 +714,7 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine { this.logger.LogTrace("(-)[NO_VOTING_DATA]"); - this.PollsRepository.SaveCurrentTip(pollsRepositoryModified ? transaction : null, chBlock.ChainedHeader.Previous); + transaction.SetTip(chBlock.ChainedHeader.Previous); return; } @@ -758,8 +746,7 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine if (targetPoll.PollVotedInFavorBlockData == new HashHeightPair(chBlock.ChainedHeader)) { this.polls.AdjustPoll(targetPoll, poll => poll.PollVotedInFavorBlockData = null); - this.PollsRepository.UpdatePoll(transaction, targetPoll); - pollsRepositoryModified = true; + transaction.UpdatePoll(targetPoll); } // Pub key of a fed member that created voting data. @@ -768,9 +755,7 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine if (voteIndex >= 0) { targetPoll.PubKeysHexVotedInFavor.RemoveAt(voteIndex); - - this.PollsRepository.UpdatePoll(transaction, targetPoll); - pollsRepositoryModified = true; + transaction.UpdatePoll(targetPoll); } } @@ -779,14 +764,13 @@ private void UnProcessBlock(DBreeze.Transactions.Transaction transaction, Chaine if (targetPoll.PollStartBlockData.Height >= chBlock.ChainedHeader.Height) { this.polls.Remove(targetPoll); - this.PollsRepository.RemovePolls(transaction, targetPoll.Id); - pollsRepositoryModified = true; + transaction.RemovePolls(targetPoll.Id); this.logger.LogDebug("Poll with Id {0} was removed.", targetPoll.Id); } } - this.PollsRepository.SaveCurrentTip(pollsRepositoryModified ? transaction : null, chBlock.ChainedHeader.Previous); + transaction.SetTip(chBlock.ChainedHeader.Previous); } } @@ -861,17 +845,16 @@ internal bool Synchronize(ChainedHeader newTip) if (headers.Count > 0) { - DBreeze.Transactions.Transaction currentTransaction = this.PollsRepository.GetTransaction(); - - int i = 0; - foreach (Block block in this.blockRepository.EnumerateBatch(headers)) + this.PollsRepository.WithTransaction((currentTransaction) => { - if (this.nodeLifetime.ApplicationStopping.IsCancellationRequested) + int i = 0; + foreach (Block block in this.blockRepository.GetBlocks(headers.Select(h => h.HashBlock).ToList())) { - this.logger.LogTrace("(-)[NODE_DISPOSED]"); - this.PollsRepository.SaveCurrentTip(currentTransaction); - currentTransaction.Commit(); - currentTransaction.Dispose(); + if (this.nodeLifetime.ApplicationStopping.IsCancellationRequested) + { + this.logger.LogTrace("(-)[NODE_DISPOSED]"); + currentTransaction.Commit(); + currentTransaction.Dispose(); bSuccess = false; return; @@ -889,23 +872,12 @@ internal bool Synchronize(ChainedHeader newTip) this.logger.LogInformation(progressString); this.signals.Publish(new FullNodeEvent() { Message = progressString, State = FullNodeState.Initializing.ToString() }); - this.PollsRepository.SaveCurrentTip(currentTransaction); - - currentTransaction.Commit(); - currentTransaction.Dispose(); - - currentTransaction = this.PollsRepository.GetTransaction(); + currentTransaction.Flush(); + } } - } - - // If we ended the synchronization at say block 10100, the current transaction would still be open and - // thus we need to commit and dispose of it. - // If we ended at block 10000, then the current transaction would have been committed and - // disposed and re-opened. - this.PollsRepository.SaveCurrentTip(currentTransaction); - currentTransaction.Commit(); - currentTransaction.Dispose(); + currentTransaction.Commit(); + }); } }); From 1bb22ca70f94c95dd65983a8e80fd4182052f404 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Apr 2022 15:39:32 +1000 Subject: [PATCH 06/44] [IBD/Rewind] Optimize commitment height rule (#931) * [IBD/Rewind] Optimize commitment height rule * Use SkipValidation * Revert constructor (cherry picked from commit d293c7e8aac1e106c414912c6d5aec2b2fe4ea00) --- .../CheckCollateralCommitmentHeightRule.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Stratis.Features.Collateral/CheckCollateralCommitmentHeightRule.cs b/src/Stratis.Features.Collateral/CheckCollateralCommitmentHeightRule.cs index 6a385bbbd9..0498221c12 100644 --- a/src/Stratis.Features.Collateral/CheckCollateralCommitmentHeightRule.cs +++ b/src/Stratis.Features.Collateral/CheckCollateralCommitmentHeightRule.cs @@ -19,6 +19,9 @@ public sealed class CheckCollateralCommitmentHeightRule : FullValidationConsensu { public override Task RunAsync(RuleContext context) { + if (context.SkipValidation) + return Task.CompletedTask; + // The genesis block won't contain any commitment data. if (context.ValidationContext.ChainedHeaderToValidate.Height == 0) return Task.CompletedTask; From 2fba69548598b506e2f891b39f3557cedb37b23f Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Apr 2022 15:40:36 +1000 Subject: [PATCH 07/44] [TODO] Make ConsensusOptions non-mutable (#885) * Resolve TODO - Make ConsensusOptions non-mutable * Update xml documentation * Do the same for PoAConsensusOptions * Make new property protected --- src/NBitcoin/ConsensusOptions.cs | 21 +++---- .../Rules/CommonRules/BlockSizeRuleTest.cs | 13 ++-- .../Rules/CommonRules/CheckSigOpsRuleTest.cs | 61 ++++++++++--------- .../TestPoANetwork.cs | 10 ++- .../PoATestsBase.cs | 10 ++- .../PoAConsensusOptions.cs | 60 +++++++++++++++--- .../PoANetwork.cs | 8 +-- .../EnforcePeerVersionCheckBehaviorTests.cs | 8 +-- src/Stratis.Sidechains.Networks/CirrusDev.cs | 8 +-- .../CirrusRegTest.cs | 10 ++- .../SmartContractsPoARegTest.cs | 8 +-- 11 files changed, 123 insertions(+), 94 deletions(-) diff --git a/src/NBitcoin/ConsensusOptions.cs b/src/NBitcoin/ConsensusOptions.cs index 1764de93b2..3afcc6a16d 100644 --- a/src/NBitcoin/ConsensusOptions.cs +++ b/src/NBitcoin/ConsensusOptions.cs @@ -4,7 +4,6 @@ namespace NBitcoin { /// /// An extension to to enable additional options to the consensus data. - /// TODO: Make immutable. /// public class ConsensusOptions { @@ -14,16 +13,16 @@ public class ConsensusOptions public const int SerializeTransactionNoWitness = 0x40000000; /// Maximum size for a block in bytes. - public uint MaxBlockBaseSize { get; set; } + public uint MaxBlockBaseSize { get; protected set; } /// The maximum allowed weight for a block, see BIP 141 (network rule) - public uint MaxBlockWeight { get; set; } + public uint MaxBlockWeight { get; protected set; } /// The maximum allowed size for a serialized block, in bytes (only for buffer size limits). - public uint MaxBlockSerializedSize { get; set; } + public uint MaxBlockSerializedSize { get; protected set; } /// Scale of witness vs other transaction data. e.g. if set to 4, then witnesses have 1/4 the weight per byte of other transaction data. - public int WitnessScaleFactor { get; set; } + public int WitnessScaleFactor { get; protected set; } /// /// Changing the default transaction version requires a two step process: @@ -33,23 +32,23 @@ public class ConsensusOptions /// will be equal. /// /// - public int MaxStandardVersion { get; set; } + public int MaxStandardVersion { get; protected set; } /// The maximum weight for transactions we're willing to relay/mine. - public int MaxStandardTxWeight { get; set; } + public int MaxStandardTxWeight { get; protected set; } /// The maximum allowed number of signature check operations in a block (network rule). - public int MaxBlockSigopsCost { get; set; } + public int MaxBlockSigopsCost { get; protected set; } /// The maximum number of sigops we're willing to relay/mine in a single tx. - public int MaxStandardTxSigopsCost { get; set; } + public int MaxStandardTxSigopsCost { get; protected set; } /// Block Height at which the node should enforce the use of . /// Can be set to zero to indicate that the minimum supported protocol version will not change depending on the block height. - public int EnforceMinProtocolVersionAtBlockHeight { get; set; } + public int EnforceMinProtocolVersionAtBlockHeight { get; protected set; } /// The minimum protocol version which should be used from block height defined in - public ProtocolVersion? EnforcedMinProtocolVersion { get; set; } + public ProtocolVersion? EnforcedMinProtocolVersion { get; protected set; } /// /// Initializes the default values. Currently only used for initialising Bitcoin networks and testing. diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/BlockSizeRuleTest.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/BlockSizeRuleTest.cs index 4e23801f52..bb510c771e 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/BlockSizeRuleTest.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/BlockSizeRuleTest.cs @@ -4,6 +4,7 @@ using NBitcoin; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; +using Stratis.Bitcoin.Tests.Common; using Xunit; namespace Stratis.Bitcoin.Features.Consensus.Tests.Rules.CommonRules @@ -50,7 +51,7 @@ public async Task RunAsync_TransactionCountAboveMaxBlockBaseSize_ThrowsBadBlockL int blockWeight = this.CalculateBlockWeight(this.ruleContext.ValidationContext.BlockToValidate, TransactionOptions.All); // increase max block weight to be able to hit this if statement - this.options.MaxBlockWeight = (uint) (blockWeight * 4) + 100; + this.options.SetPrivatePropertyValue("MaxBlockWeight", (uint) (blockWeight * 4) + 100); ConsensusErrorException exception = await Assert.ThrowsAsync(() => this.consensusRules.RegisterRule().RunAsync(this.ruleContext)); @@ -64,7 +65,7 @@ public async Task RunAsync_BlockSizeAboveMaxBlockBaseSize_ThrowsBadBlockLengthCo int blockWeight = this.CalculateBlockWeight(this.ruleContext.ValidationContext.BlockToValidate, TransactionOptions.All); // increase max block weight to be able to hit this if statement - this.options.MaxBlockWeight = (uint) (blockWeight * 4) + 1; + this.options.SetPrivatePropertyValue("MaxBlockWeight", (uint) (blockWeight * 4) + 1); ConsensusErrorException exception = await Assert.ThrowsAsync(() => this.consensusRules.RegisterRule().RunAsync(this.ruleContext)); @@ -75,7 +76,7 @@ public async Task RunAsync_BlockSizeAboveMaxBlockBaseSize_ThrowsBadBlockLengthCo public async Task RunAsync_AtBlockWeight_BelowMaxBlockBaseSize_DoesNotThrowExceptionAsync() { this.ruleContext.ValidationContext.BlockToValidate = GenerateBlockWithWeight((int)this.options.MaxBlockWeight / this.options.WitnessScaleFactor, TransactionOptions.All); - this.options.MaxBlockBaseSize = this.options.MaxBlockWeight + 1000; + this.options.SetPrivatePropertyValue("MaxBlockBaseSize", this.options.MaxBlockWeight + 1000); await this.consensusRules.RegisterRule().RunAsync(this.ruleContext); } @@ -84,7 +85,7 @@ public async Task RunAsync_AtBlockWeight_BelowMaxBlockBaseSize_DoesNotThrowExcep public async Task RunAsync_BelowBlockWeight_BelowMaxBlockBaseSize_DoesNotThrowExceptionAsync() { this.ruleContext.ValidationContext.BlockToValidate = GenerateBlockWithWeight((int)(this.options.MaxBlockWeight / this.options.WitnessScaleFactor) - 1, TransactionOptions.All); - this.options.MaxBlockBaseSize = this.options.MaxBlockWeight + 1000; + this.options.SetPrivatePropertyValue("MaxBlockBaseSize", this.options.MaxBlockWeight + 1000); await this.consensusRules.RegisterRule().RunAsync(this.ruleContext); } @@ -106,7 +107,7 @@ public async Task TaskAsync_TransactionCountBelowLimit_DoesNotThrowExceptionAsyn public async Task RunAsync_BlockAtMaxBlockBaseSize_DoesNotThrowExceptionAsync() { this.ruleContext.ValidationContext.BlockToValidate = GenerateBlockWithWeight((int)this.options.MaxBlockBaseSize, TransactionOptions.All); - this.options.MaxBlockWeight = (this.options.MaxBlockBaseSize * 4) + 1000; + this.options.SetPrivatePropertyValue("MaxBlockWeight", (this.options.MaxBlockBaseSize * 4) + 1000); await this.consensusRules.RegisterRule().RunAsync(this.ruleContext); } @@ -115,7 +116,7 @@ public async Task RunAsync_BlockAtMaxBlockBaseSize_DoesNotThrowExceptionAsync() public async Task RunAsync_BlockBelowMaxBlockBaseSize_DoesNotThrowExceptionAsync() { this.ruleContext.ValidationContext.BlockToValidate = GenerateBlockWithWeight((int)this.options.MaxBlockBaseSize - 1, TransactionOptions.All); - this.options.MaxBlockWeight = (this.options.MaxBlockBaseSize * 4) + 1000; + this.options.SetPrivatePropertyValue("MaxBlockWeight", (this.options.MaxBlockBaseSize * 4) + 1000); await this.consensusRules.RegisterRule().RunAsync(this.ruleContext); } diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/CheckSigOpsRuleTest.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/CheckSigOpsRuleTest.cs index 0f9e444b28..6c2e16efbb 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/CheckSigOpsRuleTest.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/Rules/CommonRules/CheckSigOpsRuleTest.cs @@ -2,6 +2,7 @@ using NBitcoin; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; +using Stratis.Bitcoin.Tests.Common; using Xunit; namespace Stratis.Bitcoin.Features.Consensus.Tests.Rules.CommonRules @@ -19,8 +20,8 @@ public CheckSigOpsRuleTest() [Fact] public async Task RunAsync_SingleTransactionInputSigOpsCountAboveThresHold_ThrowsBadBlockSigOpsConsensusErrorExceptionAsync() { - this.options.MaxBlockSigopsCost = 7; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 7); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -35,8 +36,8 @@ public async Task RunAsync_SingleTransactionInputSigOpsCountAboveThresHold_Throw [Fact] public async Task RunAsync_MultipleTransactionInputSigOpsCountAboveThresHold_ThrowsBadBlockSigOpsConsensusErrorExceptionAsync() { - this.options.MaxBlockSigopsCost = 7; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 7); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -52,8 +53,8 @@ public async Task RunAsync_MultipleTransactionInputSigOpsCountAboveThresHold_Thr [Fact] public async Task RunAsync_SingleTransactionOutputSigOpsCountAboveThresHold_ThrowsBadBlockSigOpsConsensusErrorExceptionAsync() { - this.options.MaxBlockSigopsCost = 7; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 7); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -68,8 +69,8 @@ public async Task RunAsync_SingleTransactionOutputSigOpsCountAboveThresHold_Thro [Fact] public async Task RunAsync_MultipleTransactionOutputSigOpsCountAboveThresHold_ThrowsBadBlockSigOpsConsensusErrorExceptionAsync() { - this.options.MaxBlockSigopsCost = 7; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 7); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -85,8 +86,8 @@ public async Task RunAsync_MultipleTransactionOutputSigOpsCountAboveThresHold_Th [Fact] public async Task RunAsync_CombinedTransactionInputOutputSigOpsCountAboveThresHold_ThrowsBadBlockSigOpsConsensusErrorExceptionAsync() { - this.options.MaxBlockSigopsCost = 7; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 7); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -102,8 +103,8 @@ public async Task RunAsync_CombinedTransactionInputOutputSigOpsCountAboveThresHo [Fact] public async Task RunAsync_SingleTransactionInputSigOpsCountAtThresHold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 8; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 8); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -116,8 +117,8 @@ public async Task RunAsync_SingleTransactionInputSigOpsCountAtThresHold_DoesNotT [Fact] public async Task RunAsync_MultipleTransactionInputSigOpsCountAtThresHold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 8; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 8); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -131,8 +132,8 @@ public async Task RunAsync_MultipleTransactionInputSigOpsCountAtThresHold_DoesNo [Fact] public async Task RunAsync_SingleTransactionOutputSigOpsCountAtThresHold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 8; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 8); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -145,8 +146,8 @@ public async Task RunAsync_SingleTransactionOutputSigOpsCountAtThresHold_DoesNot [Fact] public async Task RunAsync_MultipleTransactionOutputSigOpsCountAtThresHold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 8; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 8); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -160,8 +161,8 @@ public async Task RunAsync_MultipleTransactionOutputSigOpsCountAtThresHold_DoesN [Fact] public async Task RunAsync_CombinedTransactionInputOutputSigOpsCountAtThresHold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 8; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 8); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -175,8 +176,8 @@ public async Task RunAsync_CombinedTransactionInputOutputSigOpsCountAtThresHold_ [Fact] public async Task RunAsync_SingleTransactionInputSigOpsCountBelowThreshold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 9; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 9); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -189,8 +190,8 @@ public async Task RunAsync_SingleTransactionInputSigOpsCountBelowThreshold_DoesN [Fact] public async Task RunAsync_MultipleTransactionInputSigOpsCountBelowThreshold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 9; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 9); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -204,8 +205,8 @@ public async Task RunAsync_MultipleTransactionInputSigOpsCountBelowThreshold_Doe [Fact] public async Task RunAsync_SingleTransactionOutputSigOpsCountBelowThreshold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 9; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 9); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -218,8 +219,8 @@ public async Task RunAsync_SingleTransactionOutputSigOpsCountBelowThreshold_Does [Fact] public async Task RunAsync_MultipleTransactionOutputSigOpsCountBelowThreshold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 9; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 9); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; @@ -233,8 +234,8 @@ public async Task RunAsync_MultipleTransactionOutputSigOpsCountBelowThreshold_Do [Fact] public async Task RunAsync_CombinedTransactionInputOutputSigOpsCountBelowThreshold_DoesNotThrowExceptionAsync() { - this.options.MaxBlockSigopsCost = 9; - this.options.WitnessScaleFactor = 2; + this.options.SetPrivatePropertyValue("MaxBlockSigopsCost", 9); + this.options.SetPrivatePropertyValue("WitnessScaleFactor", 2); var transaction = new Transaction(); var op = new Op() { Code = OpcodeType.OP_CHECKSIG }; diff --git a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/TestPoANetwork.cs b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/TestPoANetwork.cs index 104e020f0d..ffec59d148 100644 --- a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/TestPoANetwork.cs +++ b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/TestPoANetwork.cs @@ -48,11 +48,9 @@ public TestPoANetwork(string networkName = "") targetSpacingSeconds: 60, votingEnabled: baseOptions.VotingEnabled, autoKickIdleMembers: false, - federationMemberMaxIdleTimeSeconds: baseOptions.FederationMemberMaxIdleTimeSeconds - ) - { - PollExpiryBlocks = 450 - }; + federationMemberMaxIdleTimeSeconds: baseOptions.FederationMemberMaxIdleTimeSeconds, + pollExpiryBlocks: 450 + ); this.Consensus.SetPrivatePropertyValue(nameof(this.Consensus.MaxReorgLength), (uint)5); } @@ -69,7 +67,7 @@ public TestPoACollateralNetwork(bool enableIdleKicking = false, string name = "" foreach (IFederationMember member in members) options.GenesisFederationMembers.Add(member); - this.ConsensusOptions.AutoKickIdleMembers = enableIdleKicking; + this.ConsensusOptions.SetPrivatePropertyValue("AutoKickIdleMembers", enableIdleKicking); this.Consensus.ConsensusRules.FullValidationRules.Add(typeof(MandatoryCollateralMemberVotingRule)); this.Consensus.MempoolRules.Add(typeof(VotingRequestValidationRule)); diff --git a/src/Stratis.Bitcoin.Features.PoA.Tests/PoATestsBase.cs b/src/Stratis.Bitcoin.Features.PoA.Tests/PoATestsBase.cs index 6eae3eb7f5..3fd08c3f46 100644 --- a/src/Stratis.Bitcoin.Features.PoA.Tests/PoATestsBase.cs +++ b/src/Stratis.Bitcoin.Features.PoA.Tests/PoATestsBase.cs @@ -189,12 +189,10 @@ public TestPoANetwork(List pubKeysOverride = null) targetSpacingSeconds: 60, votingEnabled: baseOptions.VotingEnabled, autoKickIdleMembers: baseOptions.AutoKickIdleMembers, - federationMemberMaxIdleTimeSeconds: baseOptions.FederationMemberMaxIdleTimeSeconds - ) - { - PollExpiryBlocks = 10, - Release1100ActivationHeight = 10 - }; + federationMemberMaxIdleTimeSeconds: baseOptions.FederationMemberMaxIdleTimeSeconds, + pollExpiryBlocks: 10, + release1100ActivationHeight: 10 + ); this.Consensus.SetPrivatePropertyValue(nameof(this.Consensus.MaxReorgLength), (uint)5); } diff --git a/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs b/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs index 2d93d0d1d6..36baf52ef6 100644 --- a/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs +++ b/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using NBitcoin; +using NBitcoin.Protocol; namespace Stratis.Bitcoin.Features.PoA { @@ -25,7 +26,7 @@ public class PoAConsensusOptions : ConsensusOptions /// Makes federation members kick idle members. /// Requires voting to be enabled to be set true. - public bool AutoKickIdleMembers { get; set; } + public bool AutoKickIdleMembers { get; protected set; } /// Time that federation member has to be idle to be kicked by others in case is enabled. public uint FederationMemberMaxIdleTimeSeconds { get; protected set; } @@ -33,7 +34,7 @@ public class PoAConsensusOptions : ConsensusOptions /// /// This currently only applies to Cirrus Main Net. /// - public uint? FederationMemberActivationTime { get; set; } + public uint? FederationMemberActivationTime { get; protected set; } /// /// The height at which a federation members will be resolved via the class. @@ -46,12 +47,12 @@ public class PoAConsensusOptions : ConsensusOptions /// method which resolves the pubkey from the signature directly. /// /// - public int VotingManagerV2ActivationHeight { get; set; } + public int VotingManagerV2ActivationHeight { get; protected set; } /// /// This is the height on the main chain at which the dynamic fees paid to the multsig for interop conversion requests will activate. /// - public int InterFluxV2MainChainActivationHeight { get; set; } + public int InterFluxV2MainChainActivationHeight { get; protected set; } /// /// The height at which Release 1.3.0.0 became BIP activated. @@ -71,30 +72,30 @@ public class PoAConsensusOptions : ConsensusOptions /// Legacy mining slots are determined by mining_slot = block_height % number_of_federation_members. /// Once the specified height is reached there should no longer be a shift in mining slots when new federation members are added/removed. /// - public int GetMiningTimestampV2ActivationHeight { get; set; } + public int GetMiningTimestampV2ActivationHeight { get; protected set; } /// /// The height at which inituitive mining slots are enfored without any lenience. /// Currently errors are sometimes suppressed if a federation change occurred. /// - public int GetMiningTimestampV2ActivationStrictHeight { get; set; } + public int GetMiningTimestampV2ActivationStrictHeight { get; protected set; } /// /// Logic related to release 1.1.0.0 will activate at this height, this includes Poll Expiry and the Join Federation Voting Request consensus rule. /// - public int Release1100ActivationHeight { get; set; } + public int Release1100ActivationHeight { get; protected set; } /// /// Polls are expired once the tip reaches a block this far beyond the poll start block. /// I.e. if (Math.Max(startblock + PollExpiryBlocks, PollExpiryActivationHeight) <= tip) (See IsPollExpiredAt) /// - public int PollExpiryBlocks { get; set; } + public int PollExpiryBlocks { get; protected set; } /// /// Defines when V2 of the contract serializer will be used. /// I.e if tip <= ContractSerializerV2ActivationHeight, V1 will be used. /// - public int ContractSerializerV2ActivationHeight { get; set; } + public int ContractSerializerV2ActivationHeight { get; protected set; } /// Initializes values for networks that use block size rules. /// See . @@ -107,6 +108,16 @@ public class PoAConsensusOptions : ConsensusOptions /// See . /// See . /// See . + /// See . + /// . + /// . + /// . + /// . + /// . + /// . + /// . + /// . + /// . public PoAConsensusOptions( uint maxBlockBaseSize, int maxStandardVersion, @@ -117,7 +128,17 @@ public PoAConsensusOptions( uint targetSpacingSeconds, bool votingEnabled, bool autoKickIdleMembers, - uint federationMemberMaxIdleTimeSeconds = 60 * 60 * 24 * 7) + uint federationMemberMaxIdleTimeSeconds = 60 * 60 * 24 * 7, + int? enforceMinProtocolVersionAtBlockHeight = null, + ProtocolVersion? enforcedMinProtocolVersion = null, + uint? federationMemberActivationTime = null, + int? votingManagerV2ActivationHeight = null, + int? interFluxV2MainChainActivationHeight = null, + int? getMiningTimestampV2ActivationHeight = null, + int? getMiningTimestampV2ActivationStrictHeight = null, + int? release1100ActivationHeight = null, + int? pollExpiryBlocks = null, + int? contractSerializerV2ActivationHeight = null) : base(maxBlockBaseSize, maxStandardVersion, maxStandardTxWeight, maxBlockSigopsCost, maxStandardTxSigopsCost, witnessScaleFactor: 1) { this.GenesisFederationMembers = genesisFederationMembers; @@ -126,6 +147,25 @@ public PoAConsensusOptions( this.AutoKickIdleMembers = autoKickIdleMembers; this.FederationMemberMaxIdleTimeSeconds = federationMemberMaxIdleTimeSeconds; this.InterFluxV2MainChainActivationHeight = 0; + if (enforceMinProtocolVersionAtBlockHeight.HasValue) + this.EnforceMinProtocolVersionAtBlockHeight = enforceMinProtocolVersionAtBlockHeight.Value; + if (enforcedMinProtocolVersion.HasValue) + this.EnforcedMinProtocolVersion = enforcedMinProtocolVersion.Value; + this.FederationMemberActivationTime = federationMemberActivationTime; + if (pollExpiryBlocks.HasValue) + this.PollExpiryBlocks = pollExpiryBlocks.Value; + if (interFluxV2MainChainActivationHeight.HasValue) + this.InterFluxV2MainChainActivationHeight = interFluxV2MainChainActivationHeight.Value; + if (getMiningTimestampV2ActivationHeight.HasValue) + this.GetMiningTimestampV2ActivationHeight = getMiningTimestampV2ActivationHeight.Value; + if (getMiningTimestampV2ActivationStrictHeight.HasValue) + this.GetMiningTimestampV2ActivationStrictHeight = getMiningTimestampV2ActivationStrictHeight.Value; + if (votingManagerV2ActivationHeight.HasValue) + this.VotingManagerV2ActivationHeight = votingManagerV2ActivationHeight.Value; + if (release1100ActivationHeight.HasValue) + this.Release1100ActivationHeight = release1100ActivationHeight.Value; + if (contractSerializerV2ActivationHeight.HasValue) + this.ContractSerializerV2ActivationHeight = contractSerializerV2ActivationHeight.Value; if (this.AutoKickIdleMembers && !this.VotingEnabled) throw new ArgumentException("Voting should be enabled for automatic kicking to work."); diff --git a/src/Stratis.Bitcoin.Features.PoA/PoANetwork.cs b/src/Stratis.Bitcoin.Features.PoA/PoANetwork.cs index 0680392469..47336f018d 100644 --- a/src/Stratis.Bitcoin.Features.PoA/PoANetwork.cs +++ b/src/Stratis.Bitcoin.Features.PoA/PoANetwork.cs @@ -106,11 +106,9 @@ public PoANetwork() genesisFederationMembers: genesisFederationMembers, targetSpacingSeconds: 16, votingEnabled: true, - autoKickIdleMembers: true - ) - { - PollExpiryBlocks = 450 - }; + autoKickIdleMembers: true, + pollExpiryBlocks: 450 + ); var buriedDeployments = new BuriedDeploymentsArray { diff --git a/src/Stratis.Bitcoin.Tests/P2P/EnforcePeerVersionCheckBehaviorTests.cs b/src/Stratis.Bitcoin.Tests/P2P/EnforcePeerVersionCheckBehaviorTests.cs index 525410ccb0..26eb8d0efa 100644 --- a/src/Stratis.Bitcoin.Tests/P2P/EnforcePeerVersionCheckBehaviorTests.cs +++ b/src/Stratis.Bitcoin.Tests/P2P/EnforcePeerVersionCheckBehaviorTests.cs @@ -58,8 +58,8 @@ private void Disconnected(Mock peer, string reason, Exception exce public void IncompatibileNodesDisconnectAfterHardFork() { // Set the hard-fork parameters. - this.Network.Consensus.Options.EnforceMinProtocolVersionAtBlockHeight = 5; - this.Network.Consensus.Options.EnforcedMinProtocolVersion = ProtocolVersion.CIRRUS_VERSION; + this.Network.Consensus.Options.SetPrivatePropertyValue("EnforceMinProtocolVersionAtBlockHeight", 5); + this.Network.Consensus.Options.SetPrivatePropertyValue("EnforcedMinProtocolVersion", ProtocolVersion.CIRRUS_VERSION); // Configure local node version. var nodeSettings = NodeSettings.Default(this.Network, ProtocolVersion.CIRRUS_VERSION); @@ -101,8 +101,8 @@ public void IncompatibileNodesDisconnectAfterHardFork() public void CompatibileNodesStayConnectedAfterHardFork() { // Set the hard-fork parameters. - this.Network.Consensus.Options.EnforceMinProtocolVersionAtBlockHeight = 5; - this.Network.Consensus.Options.EnforcedMinProtocolVersion = ProtocolVersion.CIRRUS_VERSION; + this.Network.Consensus.Options.SetPrivatePropertyValue("EnforceMinProtocolVersionAtBlockHeight", 5); + this.Network.Consensus.Options.SetPrivatePropertyValue("EnforcedMinProtocolVersion", ProtocolVersion.CIRRUS_VERSION); // Configure local node version. var nodeSettings = NodeSettings.Default(this.Network, ProtocolVersion.CIRRUS_VERSION); diff --git a/src/Stratis.Sidechains.Networks/CirrusDev.cs b/src/Stratis.Sidechains.Networks/CirrusDev.cs index 401e4c0101..19483278e2 100644 --- a/src/Stratis.Sidechains.Networks/CirrusDev.cs +++ b/src/Stratis.Sidechains.Networks/CirrusDev.cs @@ -91,11 +91,9 @@ public CirrusDev() genesisFederationMembers: genesisFederationMembers, targetSpacingSeconds: 16, votingEnabled: true, - autoKickIdleMembers: true - ) - { - PollExpiryBlocks = 10 - }; + autoKickIdleMembers: true, + pollExpiryBlocks: 10 + ); var buriedDeployments = new BuriedDeploymentsArray { diff --git a/src/Stratis.Sidechains.Networks/CirrusRegTest.cs b/src/Stratis.Sidechains.Networks/CirrusRegTest.cs index 42ce6aede2..b7c2c3b3d8 100644 --- a/src/Stratis.Sidechains.Networks/CirrusRegTest.cs +++ b/src/Stratis.Sidechains.Networks/CirrusRegTest.cs @@ -109,12 +109,10 @@ public CirrusRegTest() genesisFederationMembers: genesisFederationMembers, targetSpacingSeconds: 16, votingEnabled: true, - autoKickIdleMembers: true) - { - GetMiningTimestampV2ActivationHeight = 100, - GetMiningTimestampV2ActivationStrictHeight = 100, - PollExpiryBlocks = 450 // 2 hours - }; + autoKickIdleMembers: true, + getMiningTimestampV2ActivationHeight: 100, + getMiningTimestampV2ActivationStrictHeight: 100, + pollExpiryBlocks: 450); // 2 hours var buriedDeployments = new BuriedDeploymentsArray { diff --git a/src/Stratis.SmartContracts.Networks/SmartContractsPoARegTest.cs b/src/Stratis.SmartContracts.Networks/SmartContractsPoARegTest.cs index 5d32a93ac9..e3332240c9 100644 --- a/src/Stratis.SmartContracts.Networks/SmartContractsPoARegTest.cs +++ b/src/Stratis.SmartContracts.Networks/SmartContractsPoARegTest.cs @@ -65,11 +65,9 @@ public SmartContractsPoARegTest() genesisFederationMembers: genesisFederationMembers, targetSpacingSeconds: 60, votingEnabled: true, - autoKickIdleMembers: false - ) - { - PollExpiryBlocks = 450 - }; + autoKickIdleMembers: false, + pollExpiryBlocks: 450 + ); var buriedDeployments = new BuriedDeploymentsArray { From 76f005f13fad6f0f926fe2c9bf2a2e513d2f1d5b Mon Sep 17 00:00:00 2001 From: quantumagi Date: Sat, 30 Apr 2022 20:56:36 +1000 Subject: [PATCH 08/44] Fix merge (#967) (cherry picked from commit 78c5311347b9f5aa26e554b576571e86df10f8ef) --- .../Voting/VotingManagerProcessBlock.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManagerProcessBlock.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManagerProcessBlock.cs index d3e9c0d501..a100dc4f6d 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManagerProcessBlock.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManagerProcessBlock.cs @@ -11,9 +11,9 @@ namespace Stratis.Bitcoin.Features.PoA.Voting public sealed class VotingManagerProcessBlock : EventBase { public ChainedHeaderBlock ConnectedBlock { get; } - public DBreeze.Transactions.Transaction PollsRepositoryTransaction { get; } + public PollsRepository.Transaction PollsRepositoryTransaction { get; } - public VotingManagerProcessBlock(ChainedHeaderBlock connectedBlock, DBreeze.Transactions.Transaction pollsRepositoryTransaction = null) + public VotingManagerProcessBlock(ChainedHeaderBlock connectedBlock, PollsRepository.Transaction pollsRepositoryTransaction = null) { this.ConnectedBlock = connectedBlock; this.PollsRepositoryTransaction = pollsRepositoryTransaction; From c29ac8f5093f9437c2b3001c83ce3d4fb2ef5fd6 Mon Sep 17 00:00:00 2001 From: zeptin Date: Sun, 15 May 2022 20:22:20 +0200 Subject: [PATCH 09/44] [API] Add hex version of getblockheader (#983) * Add hex version of getblockheader * Fix test (cherry picked from commit 1bce97ba56939052215a05d8628261f20c26823b) --- .../NodeController.cs | 19 ++++++++----------- .../Controllers/NodeControllerTest.cs | 17 ++++++++--------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Api/NodeController.cs b/src/Stratis.Bitcoin.Features.Api/NodeController.cs index cbd28eb925..6c73bac184 100644 --- a/src/Stratis.Bitcoin.Features.Api/NodeController.cs +++ b/src/Stratis.Bitcoin.Features.Api/NodeController.cs @@ -219,11 +219,10 @@ public IActionResult Status([FromQuery] bool publish) /// Gets the block header of a block identified by a block hash. /// /// The hash of the block to retrieve. - /// A flag that specifies whether to return the block header in the JSON format. Defaults to true. A value of false is currently not supported. - /// Json formatted . null if block not found. Returns formatted error if fails. - /// Returns the blockheader if found. - /// Null hash provided, BlockHeader does not exist or if isJsonFormat = false>/response> - /// Binary serialization is not supported with this method. + /// A flag that specifies whether to return the block header in the JSON format. Defaults to true. A value of false will return the header serialized to hexadecimal format. + /// Json formatted . Returns if Json format is not requested. null if block not found. Returns formatted error if fails. + /// Returns the block header if found. + /// Null hash provided, or block header does not exist. [Route("getblockheader")] [HttpGet] [ProducesResponseType((int)HttpStatusCode.OK)] @@ -238,16 +237,14 @@ public IActionResult GetBlockHeader([FromQuery] string hash, bool isJsonFormat = this.logger.LogDebug("GetBlockHeader {0}", hash); - if (!isJsonFormat) - { - this.logger.LogError("Binary serialization is not supported."); - return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, "Error", "Binary serialization is not supported."); - } - BlockHeader blockHeader = this.chainIndexer?.GetHeader(uint256.Parse(hash))?.Header; + if (blockHeader == null) return this.NotFound($"Block header for '{hash}' not found"); + if (!isJsonFormat) + return this.Json(new HexModel(blockHeader.ToHex(this.network))); + return this.Json(new BlockHeaderModel(blockHeader)); } catch (Exception e) diff --git a/src/Stratis.Bitcoin.Tests/Controllers/NodeControllerTest.cs b/src/Stratis.Bitcoin.Tests/Controllers/NodeControllerTest.cs index 8b6c538072..9024f631f8 100644 --- a/src/Stratis.Bitcoin.Tests/Controllers/NodeControllerTest.cs +++ b/src/Stratis.Bitcoin.Tests/Controllers/NodeControllerTest.cs @@ -514,19 +514,18 @@ public async Task GetTxOutAsync_IncludeInMempool_UnspentTransactionFound_Returns } [Fact] - public void GetBlockHeader_NotUsingJsonFormat_ThrowsNotImplementedException() + public void GetBlockHeader_NotUsingJsonFormat_ReturnsHexModel() { - string hash = "1341323442"; + ChainedHeader block = this.chainIndexer.GetHeader(2); + string bits = GetBlockHeaderBits(block.Header); + string hash = block.HashBlock.ToString(); bool isJsonFormat = false; - IActionResult result = this.controller.GetBlockHeader(hash, isJsonFormat); + var json = (JsonResult)this.controller.GetBlockHeader(hash, isJsonFormat); + var resultModel = (HexModel)json.Value; - var errorResult = Assert.IsType(result); - var errorResponse = Assert.IsType(errorResult.Value); - Assert.Single(errorResponse.Errors); - ErrorModel error = errorResponse.Errors[0]; - Assert.Equal(400, error.Status); - Assert.StartsWith("Binary serialization is not", error.Description); + Assert.NotNull(resultModel); + Assert.Equal(block.Header.ToHex(this.network), resultModel.Hex); } [Fact] From 949e96d7c6e71f288fa658d4c4df20ce841181df Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 31 May 2022 17:43:01 +1000 Subject: [PATCH 10/44] Add BaseSettings class (#991) (cherry picked from commit d99b00eb29fbfe1e2a5066a9a778c8417f25d157) --- .../LightWalletFeature.cs | 3 +- .../WalletFeature.cs | 5 +- .../WalletSettings.cs | 83 ++-------- .../Configuration/Settings/BaseSettings.cs | 144 ++++++++++++++++++ 4 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs diff --git a/src/Stratis.Bitcoin.Features.LightWallet/LightWalletFeature.cs b/src/Stratis.Bitcoin.Features.LightWallet/LightWalletFeature.cs index d17e1cf2bf..7208e35e64 100644 --- a/src/Stratis.Bitcoin.Features.LightWallet/LightWalletFeature.cs +++ b/src/Stratis.Bitcoin.Features.LightWallet/LightWalletFeature.cs @@ -12,6 +12,7 @@ using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Builder.Feature; using Stratis.Bitcoin.Configuration.Logging; +using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Connection; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.BlockStore; @@ -110,7 +111,7 @@ public LightWalletFeature( /// The network to extract values from. public static void PrintHelp(Network network) { - WalletSettings.PrintHelp(network); + BaseSettings.PrintHelp(typeof(WalletSettings), network); } /// diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs index d459f9ffdd..d539f9d203 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs @@ -8,6 +8,7 @@ using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Builder.Feature; using Stratis.Bitcoin.Configuration.Logging; +using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Connection; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.BlockStore; @@ -82,7 +83,7 @@ public WalletFeature( /// The network to extract values from. public static void PrintHelp(Network network) { - WalletSettings.PrintHelp(network); + BaseSettings.PrintHelp(typeof(WalletSettings), network); } /// @@ -92,7 +93,7 @@ public static void PrintHelp(Network network) /// The network to base the defaults off. public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) { - WalletSettings.BuildDefaultConfigurationFile(builder, network); + BaseSettings.BuildDefaultConfigurationFile(typeof(WalletSettings), builder, network); } private void AddInlineStats(StringBuilder log) diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletSettings.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletSettings.cs index 9324644f47..8cfed5aac8 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletSettings.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletSettings.cs @@ -1,38 +1,36 @@ -using System.Text; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; +using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Configuration.Settings; namespace Stratis.Bitcoin.Features.Wallet { /// /// Configuration related to the wallet. /// - public class WalletSettings + public class WalletSettings : BaseSettings { - /// Instance logger. - private readonly ILogger logger; - /// /// A value indicating whether the transactions hex representations should be saved in the wallet file. /// - public bool SaveTransactionHex { get; set; } + [CommandLineOption("savetrxhex", "Save the hex of transactions in the wallet file.")] + public bool SaveTransactionHex { get; private set; } = false; /// /// A value indicating whether to unlock the supplied default wallet on startup. /// - public bool UnlockDefaultWallet { get; set; } + [CommandLineOption("unlockdefaultwallet", "Unlocks the specified default wallet.")] + public bool UnlockDefaultWallet { get; private set; } = false; /// /// Name for the default wallet. /// - public string DefaultWalletName { get; set; } + [CommandLineOption("defaultwalletname", "Loads the specified wallet on startup. If it doesn't exist, it will be created automatically.")] + public string DefaultWalletName { get; private set; } = null; /// /// Password for the default wallet if overriding the default. /// - public string DefaultWalletPassword { get; set; } + [CommandLineOption("defaultwalletpassword", "Overrides the default wallet password.", false)] + public string DefaultWalletPassword { get; private set; } = "default"; /// /// A value indicating whether the wallet being run is the light wallet or the full wallet. @@ -40,29 +38,15 @@ public class WalletSettings public bool IsLightWallet { get; set; } /// Size of the buffer of unused addresses maintained in an account. - public int UnusedAddressesBuffer { get; set; } + [CommandLineOption("walletaddressbuffer", "Size of the buffer of unused addresses maintained in an account.")] + public int UnusedAddressesBuffer { get; private set; } = 20; /// /// Initializes an instance of the object from the node configuration. /// /// The node configuration. - public WalletSettings(NodeSettings nodeSettings) + public WalletSettings(NodeSettings nodeSettings) : base(nodeSettings) { - Guard.NotNull(nodeSettings, nameof(nodeSettings)); - - this.logger = nodeSettings.LoggerFactory.CreateLogger(typeof(WalletSettings).FullName); - - TextFileConfiguration config = nodeSettings.ConfigReader; - - this.SaveTransactionHex = config.GetOrDefault("savetrxhex", false, this.logger); - this.UnusedAddressesBuffer = config.GetOrDefault("walletaddressbuffer", 20, this.logger); - this.DefaultWalletName = config.GetOrDefault("defaultwalletname", null, this.logger); - - if (!string.IsNullOrEmpty(this.DefaultWalletName)) - { - this.DefaultWalletPassword = config.GetOrDefault("defaultwalletpassword", "default", null); // No logging! - this.UnlockDefaultWallet = config.GetOrDefault("unlockdefaultwallet", false, this.logger); - } } /// @@ -73,44 +57,5 @@ public bool IsDefaultWalletEnabled() { return !string.IsNullOrWhiteSpace(this.DefaultWalletName); } - - /// - /// Displays wallet configuration help information on the console. - /// - /// Not used. - public static void PrintHelp(Network network) - { - NodeSettings defaults = NodeSettings.Default(network); - var builder = new StringBuilder(); - - builder.AppendLine("-savetrxhex=<0 or 1> Save the hex of transactions in the wallet file. Default: 0."); - builder.AppendLine("-defaultwalletname= Loads the specified wallet on startup. If it doesn't exist, it will be created automatically."); - builder.AppendLine("-defaultwalletpassword= Overrides the default wallet password. Default: default."); - builder.AppendLine("-unlockdefaultwallet=<0 or 1> Unlocks the specified default wallet. Default: 0."); - builder.AppendLine("-walletaddressbuffer= Size of the buffer of unused addresses maintained in an account. Default: 20."); - - var logger = defaults.LoggerFactory.CreateLogger(typeof(WalletSettings).FullName); - logger.LogInformation(builder.ToString()); - } - - /// - /// Get the default configuration. - /// - /// The string builder to add the settings to. - /// The network to base the defaults off. - public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) - { - builder.AppendLine("####Wallet Settings####"); - builder.AppendLine("#Save the hex of transactions in the wallet file. Default: 0."); - builder.AppendLine("#savetrxhex=0"); - builder.AppendLine("#Creates a wallet with the specified name and the specified password. It will be created if it doesn't exist and can be unlocked on startup when unlockdefaultwallet is set to 1."); - builder.AppendLine("#defaultwalletname="); - builder.AppendLine("#Overrides the default wallet password. Default: default."); - builder.AppendLine("#defaultwalletpassword=default"); - builder.AppendLine("#A value indicating whether to unlock the supplied default wallet on startup. Default 0."); - builder.AppendLine("#unlockdefaultwallet=0"); - builder.AppendLine("#Size of the buffer of unused addresses maintained in an account. Default: 20."); - builder.AppendLine("#walletaddressbuffer=20"); - } } } diff --git a/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs new file mode 100644 index 0000000000..04a09963a3 --- /dev/null +++ b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs @@ -0,0 +1,144 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using Microsoft.Extensions.Logging; +using NBitcoin; +using Stratis.Bitcoin.Utilities; + +namespace Stratis.Bitcoin.Configuration.Settings +{ + [AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] + public class CommandLineOptionAttribute : Attribute + { + public string Option { get; private set; } + + public string Description { get; private set; } + + public object DefaultValue { get; private set; } + + public bool CanLog { get; private set; } + + public CommandLineOptionAttribute(string option, string description, bool canLog = true) + { + this.Option = option; + this.Description = description; + this.CanLog = canLog; + } + } + + public abstract class BaseSettings + { + protected readonly NodeSettings nodeSettings; + + private static string TypeDescription(Type type) + { + if (type == typeof(bool)) + return "0 or 1"; + + if (type == typeof(int)) + return "number"; + + return type.ToString().Split('.').Last().ToLower(); + } + + private static string DefaultValue(object value, bool raw = false) + { + if (value == null) + return string.Empty; + + if (value.GetType() == typeof(bool)) + value = ((bool)value) ? "1" : "0"; + + return raw ? value.ToString() : $" Default: {value}."; + } + + private static Dictionary typeGetter = new Dictionary(); + + public BaseSettings(NodeSettings nodeSettings) + { + Guard.NotNull(nodeSettings, nameof(nodeSettings)); + + this.nodeSettings = nodeSettings; + + ILogger logger = nodeSettings.LoggerFactory.CreateLogger(this.GetType().FullName); + + TextFileConfiguration config = nodeSettings.ConfigReader; + + foreach (PropertyInfo pi in this.GetType().GetProperties()) + { + CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); + if (attr == null) + continue; + + lock (typeGetter) + { + if (!typeGetter.TryGetValue(pi.PropertyType, out MethodInfo getOrDefault)) + { + getOrDefault = typeof(TextFileConfiguration).GetMethod(nameof(TextFileConfiguration.GetOrDefault)).MakeGenericMethod(pi.PropertyType); + typeGetter[pi.PropertyType] = getOrDefault; + } + + pi.SetValue(this, getOrDefault.Invoke(config, new object[] { attr.Option, pi.GetValue(this), attr.CanLog ? logger : null })); + } + } + } + + /// + /// Displays configuration help information on the console. + /// + /// The type of the class defining the settings. + /// The network. + public static void PrintHelp(Type settingsType, Network network) + { + NodeSettings defaultSettings = NodeSettings.Default(network); + var builder = new StringBuilder(); + var defaultValues = Activator.CreateInstance(settingsType, new object[] { defaultSettings }); + + foreach (PropertyInfo pi in settingsType.GetProperties()) + { + CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); + if (attr == null) + continue; + + object defaultValue = pi.GetValue(defaultValues); + + string option = $"{attr.Option}=<{TypeDescription(pi.PropertyType)}>"; + + builder.AppendLine($"-{option.PadRight(29)} {attr.Description}{DefaultValue(defaultValue)}"); + } + + var logger = defaultSettings.LoggerFactory.CreateLogger(settingsType.FullName); + logger.LogInformation(builder.ToString()); + } + + /// + /// Get the default configuration. + /// + /// The type of the class defining the settings. + /// The string builder to add the settings to. + /// The network to base the defaults off. + public static void BuildDefaultConfigurationFile(Type settingsType, StringBuilder builder, Network network) + { + builder.AppendLine($"####{settingsType.FullName}####"); + + NodeSettings defaultSettings = NodeSettings.Default(network); + var defaultValues = Activator.CreateInstance(settingsType, new object[] { defaultSettings }); + + foreach (PropertyInfo pi in settingsType.GetProperties()) + { + CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); + if (attr == null) + continue; + + object defaultValue = pi.GetValue(defaultValues); + + string option = $"{attr.Option}={DefaultValue(defaultValue, true)}"; + + builder.AppendLine($"#{attr.Description}"); + builder.AppendLine($"#{option}"); + } + } + } +} \ No newline at end of file From fc64ab6f81f3fa66d438ea9796fe4203d2f55969 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 31 May 2022 17:43:25 +1000 Subject: [PATCH 11/44] Add GetBlocks and CommonBlock APIs for chain synchronisation (#985) * Add GetBlocks and CommonBlock APIs for chain synchronisation * Fix typo * Change method * Add tests (cherry picked from commit 4b497d9eade96b7837639b7b14a50e276f91d195) --- .../BlockStoreControllerTests.cs | 32 ++++++- .../Controllers/BlockStoreController.cs | 88 +++++++++++++++---- .../Models/SearchByHeightRequest.cs | 29 ++++++ .../ConsensusController.cs | 26 ++++++ .../RPC/ConsensusActionTests.cs | 23 +++++ 5 files changed, 180 insertions(+), 18 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.BlockStore/Models/SearchByHeightRequest.cs diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs index 55730a8d4b..b9c4cb9a70 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs @@ -1,4 +1,6 @@ -using System.ComponentModel.DataAnnotations; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; using System.Net; using FluentAssertions; using Microsoft.AspNetCore.Mvc; @@ -171,6 +173,34 @@ public void GetBlockCount_ReturnsHeightFromChainState() Assert.Equal(2, result); } + [Fact] + public void Get_Blocks_Should_Return_Blocks() + { + var logger = new Mock(); + var store = new Mock(); + var chainState = new Mock(); + var addressIndexer = new Mock(); + var utxoIndexer = new Mock(); + var scriptAddressReader = Mock.Of(); + + ChainIndexer chainIndexer = WalletTestsHelpers.GenerateChainWithHeight(3, new StraxTest()); + + logger.Setup(l => l.CreateLogger(It.IsAny())).Returns(Mock.Of); + + chainState.Setup(c => c.ConsensusTip) + .Returns(chainIndexer.GetHeader(2)); + + store.Setup(s => s.GetBlocks(It.IsAny>())).Returns((List hashes) => hashes.Select(h => chainIndexer[h].Block).ToList()); + + var controller = new BlockStoreController(new StraxMain(), logger.Object, store.Object, chainState.Object, chainIndexer, addressIndexer.Object, utxoIndexer.Object, scriptAddressReader); + + var json = (JsonResult)controller.GetBlocks(new SearchByHeightRequest() { Height = 2, NumberOfBlocks = 2 }); + var jsonResult = Assert.IsType>(json.Value); + + Assert.Equal(2, jsonResult.Count); + + } + private static (Mock store, BlockStoreController controller) GetControllerAndStore() { var logger = new Mock(); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs index bd9cb04ab4..8573881532 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs @@ -26,6 +26,7 @@ public static class BlockStoreRouteEndPoint public const string VerboseAddressesBalances = "verboseaddressesbalances"; public const string GetAddressIndexerTip = "addressindexertip"; public const string GetBlock = "block"; + public const string GetBlocks = "blocks"; public const string GetBlockCount = "getblockcount"; public const string GetUtxoSet = "getutxoset"; public const string GetUtxoSetForAddress = "getutxosetforaddress"; @@ -150,29 +151,55 @@ public IActionResult GetBlock([FromQuery] SearchByHashRequest query) return this.Json(block); } - BlockModel blockModel = query.ShowTransactionDetails - ? new BlockTransactionDetailsModel(block, chainedHeader, this.chainIndexer.Tip, this.network) - : new BlockModel(block, chainedHeader, this.chainIndexer.Tip, this.network); + return this.Json(GetBlockModel(block, blockId, chainedHeader, query.ShowTransactionDetails)); + } + catch (Exception e) + { + this.logger.LogError("Exception occurred: {0}", e.ToString()); + return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, e.Message, e.ToString()); + } + } - if (this.network.Consensus.IsProofOfStake) - { - var posBlock = block as PosBlock; - blockModel.PosBlockSignature = posBlock.BlockSignature.ToHex(this.network); - blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockTarget()).ToUInt256().ToString(); - blockModel.PosChainTrust = chainedHeader.ChainWork.ToString(); // this should be similar to ChainWork + /// + /// Retrieves the blocks from a given height onwards. + /// + /// An object containing the necessary parameters to search for a block. + /// if block is found, if not found. Returns with error information if exception thrown. + /// Returns data about the block or block not found message + /// Block hash invalid, or an unexpected exception occurred + [Route(BlockStoreRouteEndPoint.GetBlocks)] + [HttpGet] + [ProducesResponseType((int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.BadRequest)] + [ProducesResponseType((int)HttpStatusCode.NotFound)] + public IActionResult GetBlocks([FromQuery] SearchByHeightRequest query) + { + if (!this.ModelState.IsValid) + return ModelStateErrors.BuildErrorResponse(this.ModelState); + + try + { + if (query.Height > this.chainIndexer.Tip.Height) + return this.NotFound("No blocks found"); - if (this.stakeChain != null) - { - BlockStake blockStake = this.stakeChain.Get(blockId); + int tip = query.Height + query.NumberOfBlocks - 1; + if (tip > this.chainIndexer.Tip.Height) + { + query.NumberOfBlocks -= (this.chainIndexer.Tip.Height - tip); + tip = this.chainIndexer.Tip.Height; + } - blockModel.PosModifierv2 = blockStake?.StakeModifierV2.ToString(); - blockModel.PosFlags = blockStake?.Flags == BlockFlag.BLOCK_PROOF_OF_STAKE ? "proof-of-stake" : "proof-of-work"; - blockModel.PosHashProof = blockStake?.HashProof?.ToString(); - } + List chainedHeaders = this.chainIndexer[tip].EnumerateToGenesis().Take(query.NumberOfBlocks).Reverse().ToList(); + List blockIds = chainedHeaders.Select(h => h.HashBlock).ToList(); + List blocks = this.blockStore.GetBlocks(blockIds); + + if (!query.OutputJson) + { + return this.Json(blocks); } - return this.Json(blockModel); + return this.Json(chainedHeaders.Select((h, n) => GetBlockModel(blocks[n], blockIds[n], h, query.ShowTransactionDetails))); } catch (Exception e) { @@ -181,6 +208,33 @@ public IActionResult GetBlock([FromQuery] SearchByHashRequest query) } } + private BlockModel GetBlockModel(Block block, uint256 blockId, ChainedHeader chainedHeader, bool showTransactionDetails) + { + BlockModel blockModel = showTransactionDetails + ? new BlockTransactionDetailsModel(block, chainedHeader, this.chainIndexer.Tip, this.network) + : new BlockModel(block, chainedHeader, this.chainIndexer.Tip, this.network); + + if (this.network.Consensus.IsProofOfStake) + { + var posBlock = block as PosBlock; + + blockModel.PosBlockSignature = posBlock.BlockSignature.ToHex(this.network); + blockModel.PosBlockTrust = new Target(chainedHeader.GetBlockTarget()).ToUInt256().ToString(); + blockModel.PosChainTrust = chainedHeader.ChainWork.ToString(); // this should be similar to ChainWork + + if (this.stakeChain != null) + { + BlockStake blockStake = this.stakeChain.Get(blockId); + + blockModel.PosModifierv2 = blockStake?.StakeModifierV2.ToString(); + blockModel.PosFlags = blockStake?.Flags == BlockFlag.BLOCK_PROOF_OF_STAKE ? "proof-of-stake" : "proof-of-work"; + blockModel.PosHashProof = blockStake?.HashProof?.ToString(); + } + } + + return blockModel; + } + /// /// Gets the current consensus tip height. /// diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Models/SearchByHeightRequest.cs b/src/Stratis.Bitcoin.Features.BlockStore/Models/SearchByHeightRequest.cs new file mode 100644 index 0000000000..1c85d0830d --- /dev/null +++ b/src/Stratis.Bitcoin.Features.BlockStore/Models/SearchByHeightRequest.cs @@ -0,0 +1,29 @@ +using System.ComponentModel.DataAnnotations; + +namespace Stratis.Bitcoin.Features.BlockStore.Models +{ + /// + /// A class containing the necessary parameters for a block search request. + /// + public class SearchByHeightRequest : RequestBase + { + /// + /// The height of the required block(s). + /// + [Required] + public int Height { get; set; } + + /// + /// The maximum number of the blocks to return. + /// + [Required] + public int NumberOfBlocks { get; set; } + + /// + /// A flag that indicates whether to return each block transaction complete with details + /// or simply return transaction hashes (TX IDs). + /// + /// This flag is not used when is set to false. + public bool ShowTransactionDetails { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.Consensus/ConsensusController.cs b/src/Stratis.Bitcoin.Features.Consensus/ConsensusController.cs index 91262e43c9..196229ab0e 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/ConsensusController.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/ConsensusController.cs @@ -50,6 +50,32 @@ public uint256 GetBestBlockHashRPC() return this.ChainState.ConsensusTip?.HashBlock; } + /// + /// Finds the first block in common with the list of provided hashes. Include the genesis hash. + /// + /// The list of provided hashes. + /// A derived from a object. + [Route("api/[controller]/commonblock")] + [HttpPost] + [ProducesResponseType((int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.BadRequest)] + public IActionResult CommonBlock([FromBody] uint256[] blockLocator) + { + try + { + ChainedHeader commonHeader = this.ChainIndexer.FindFork(blockLocator); + if (commonHeader == null) + throw new BlockNotFoundException($"No common block found"); + + return this.Json(new HashHeightPair(commonHeader)); + } + catch (Exception e) + { + this.logger.LogError("Exception occurred: {0}", e.ToString()); + return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, e.Message, e.ToString()); + } + } + /// /// Get the threshold states of softforks currently being deployed. /// Allowable states are: Defined, Started, LockedIn, Failed, Active. diff --git a/src/Stratis.Bitcoin.IntegrationTests/RPC/ConsensusActionTests.cs b/src/Stratis.Bitcoin.IntegrationTests/RPC/ConsensusActionTests.cs index 4436ddf086..7a221ba797 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/RPC/ConsensusActionTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/RPC/ConsensusActionTests.cs @@ -5,6 +5,9 @@ using Xunit; using Stratis.Bitcoin.Base; using Stratis.Bitcoin.Tests.Common; +using Microsoft.AspNetCore.Mvc; +using Stratis.Bitcoin.Utilities.JsonErrors; +using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.IntegrationTests.RPC { @@ -49,5 +52,25 @@ public void CanCall_IsInitialBlockDownload() Assert.NotNull(isIBDProvider); Assert.True(isIBDProvider.IsInitialBlockDownload()); } + + [Fact] + public void CanCall_CommonBlock() + { + string dir = CreateTestDir(this); + + IFullNode fullNode = this.BuildServicedNode(dir); + var controller = fullNode.NodeController(); + + IActionResult result = controller.CommonBlock(new[] { uint256.Zero, fullNode.Network.GenesisHash, uint256.One }); + + Assert.NotNull(result); + + var jsonResult = Assert.IsType(result); + + var hashHeightPair = Assert.IsType(jsonResult.Value); + + Assert.Equal(hashHeightPair.Height, 0); + Assert.Equal(hashHeightPair.Hash, fullNode.Network.GenesisHash); + } } } From ca700cf093ac3e1ccd9fbe6350ee35365668d7b5 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 31 May 2022 21:53:13 +1000 Subject: [PATCH 12/44] Refactor MinerSettings (#992) * Refactor MinerSettings * Fix tests (cherry picked from commit bd4664ad7804632a6556c1cce73ae340020570a0) --- .../MinerSettings.cs | 129 +++++------------- .../MiningFeature.cs | 4 +- .../Configuration/Settings/BaseSettings.cs | 10 +- .../Configuration/TextFileConfiguration.cs | 5 + 4 files changed, 50 insertions(+), 98 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs b/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs index 5cbe5c1c0c..317171e175 100644 --- a/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs +++ b/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs @@ -1,32 +1,29 @@ -using System.Text; -using Microsoft.Extensions.Logging; -using NBitcoin; +using NBitcoin; using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; +using Stratis.Bitcoin.Configuration.Settings; namespace Stratis.Bitcoin.Features.Miner { /// /// Configuration related to the miner interface. /// - public class MinerSettings + public class MinerSettings : BaseSettings { private const ulong MinimumSplitCoinValueDefaultValue = 100 * Money.COIN; private const ulong MinimumStakingCoinValueDefaultValue = 10 * Money.CENT; - /// Instance logger. - private readonly ILogger logger; - /// /// Enable the node to stake. /// - public bool Stake { get; private set; } + [CommandLineOption("stake", "Enable POS.")] + public bool Stake { get; private set; } = false; /// /// Enable splitting coins when staking. /// - public bool EnableCoinStakeSplitting { get; private set; } + [CommandLineOption("enablecoinstakesplitting", "Enable splitting coins when staking.")] + public bool EnableCoinStakeSplitting { get; private set; } = true; /// /// Minimum value a coin has to be in order to be considered for staking. @@ -34,32 +31,48 @@ public class MinerSettings /// /// This can be used to save on CPU consumption by excluding small coins that would not significantly impact a wallet's staking power. /// - public ulong MinimumStakingCoinValue { get; private set; } + [CommandLineOption("minimumstakingcoinvalue", "Minimum size of the coins considered for staking, in satoshis.")] + public ulong MinimumStakingCoinValue { get { return this.minimumStakingCoinValue; } private set { this.minimumStakingCoinValue = (value == 0) ? 1 : value; } } + private ulong minimumStakingCoinValue = MinimumStakingCoinValueDefaultValue; /// /// Targeted minimum value of staking coins after splitting. /// - public ulong MinimumSplitCoinValue { get; private set; } + [CommandLineOption("minimumsplitcoinvalue", "Targeted minimum value of staking coins after splitting, in satoshis.")] + public ulong MinimumSplitCoinValue { get; private set; } = MinimumSplitCoinValueDefaultValue; /// /// Enable the node to mine. /// - public bool Mine { get; private set; } + [CommandLineOption("mine", "Enable POW mining.")] + + public bool Mine { get; private set; } = false; /// /// An address to use when mining, if not specified an address from the wallet will be used. /// - public string MineAddress { get; set; } + [CommandLineOption("mineaddress", "The address to use for mining (empty string to select an address from the wallet).")] + public string MineAddress { get; set; } = null; /// /// The wallet password needed when staking to sign blocks. /// - public string WalletPassword { get; set; } + [CommandLineOption("walletpassword", "Password to unlock the wallet.", false)] + public string WalletPassword { get; set; } = null; /// /// The wallet name to select outputs to stake. /// - public string WalletName { get; set; } + [CommandLineOption("walletname", "The wallet name to use when staking.", false)] + public string WalletName { get; set; } = null; + + [CommandLineOption("blockmaxsize", "Maximum block size (in bytes) for the miner to generate.")] + private uint BlockMaxSize { get { return this.blockMaxSize ?? this.nodeSettings.Network.Consensus.Options.MaxBlockSerializedSize; } set { this.blockMaxSize = value; } } + private uint? blockMaxSize = null; + + [CommandLineOption("blockmaxweight", "Maximum block weight (in weight units) for the miner to generate.")] + private uint BlockMaxWeight { get { return this.blockMaxWeight ?? this.nodeSettings.Network.Consensus.Options.MaxBlockWeight; } set { this.blockMaxWeight = value; } } + private uint? blockMaxWeight = null; /// /// Settings for . @@ -70,88 +83,18 @@ public class MinerSettings /// Initializes an instance of the object from the node configuration. /// /// The node configuration. - public MinerSettings(NodeSettings nodeSettings) + public MinerSettings(NodeSettings nodeSettings) : base(nodeSettings) { - Guard.NotNull(nodeSettings, nameof(nodeSettings)); - - this.logger = nodeSettings.LoggerFactory.CreateLogger(typeof(MinerSettings).FullName); + this.BlockDefinitionOptions = new BlockDefinitionOptions(this.BlockMaxWeight, this.BlockMaxSize).RestrictForNetwork(nodeSettings.Network); - TextFileConfiguration config = nodeSettings.ConfigReader; + if (!this.Mine) + this.MineAddress = null; - this.Mine = config.GetOrDefault("mine", false, this.logger); - if (this.Mine) - this.MineAddress = config.GetOrDefault("mineaddress", null, this.logger); - - this.Stake = config.GetOrDefault("stake", false, this.logger); - if (this.Stake) + if (!this.Stake) { - this.WalletName = config.GetOrDefault("walletname", null, this.logger); - this.WalletPassword = config.GetOrDefault("walletpassword", null); // No logging! + this.WalletName = null; + this.WalletPassword = null; } - - uint blockMaxSize = (uint) config.GetOrDefault("blockmaxsize", (int) nodeSettings.Network.Consensus.Options.MaxBlockSerializedSize, this.logger); - uint blockMaxWeight = (uint) config.GetOrDefault("blockmaxweight", (int) nodeSettings.Network.Consensus.Options.MaxBlockWeight, this.logger); - - this.BlockDefinitionOptions = new BlockDefinitionOptions(blockMaxWeight, blockMaxSize).RestrictForNetwork(nodeSettings.Network); - - this.EnableCoinStakeSplitting = config.GetOrDefault("enablecoinstakesplitting", true, this.logger); - this.MinimumSplitCoinValue = config.GetOrDefault("minimumsplitcoinvalue", MinimumSplitCoinValueDefaultValue, this.logger); - this.MinimumStakingCoinValue = config.GetOrDefault("minimumstakingcoinvalue", MinimumStakingCoinValueDefaultValue, this.logger); - this.MinimumStakingCoinValue = this.MinimumStakingCoinValue == 0 ? 1 : this.MinimumStakingCoinValue; - } - - /// - /// Displays mining help information on the console. - /// - /// Not used. - public static void PrintHelp(Network network) - { - NodeSettings defaults = NodeSettings.Default(network); - var builder = new StringBuilder(); - - builder.AppendLine("-mine=<0 or 1> Enable POW mining."); - builder.AppendLine("-stake=<0 or 1> Enable POS."); - builder.AppendLine("-mineaddress= The address to use for mining (empty string to select an address from the wallet)."); - builder.AppendLine("-walletname= The wallet name to use when staking."); - builder.AppendLine("-walletpassword= Password to unlock the wallet."); - builder.AppendLine("-blockmaxsize= Maximum block size (in bytes) for the miner to generate."); - builder.AppendLine("-blockmaxweight= Maximum block weight (in weight units) for the miner to generate."); - builder.AppendLine("-enablecoinstakesplitting=<0 or 1> Enable splitting coins when staking. This is true by default."); - builder.AppendLine($"-minimumstakingcoinvalue= Minimum size of the coins considered for staking, in satoshis. Default value is {MinimumStakingCoinValueDefaultValue:N0} satoshis (= {MinimumStakingCoinValueDefaultValue / (decimal)Money.COIN:N1} Coin)."); - builder.AppendLine($"-minimumsplitcoinvalue= Targeted minimum value of staking coins after splitting, in satoshis. Default value is {MinimumSplitCoinValueDefaultValue:N0} satoshis (= {MinimumSplitCoinValueDefaultValue / Money.COIN} Coin)."); - - var logger = NodeSettings.Default(network).LoggerFactory.CreateLogger(typeof(MinerSettings).FullName); - logger.LogInformation(builder.ToString()); - } - - /// - /// Get the default configuration. - /// - /// The string builder to add the settings to. - /// The network to base the defaults off. - public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) - { - builder.AppendLine("####Miner Settings####"); - builder.AppendLine("#Enable POW mining."); - builder.AppendLine("#mine=0"); - builder.AppendLine("#Enable POS."); - builder.AppendLine("#stake=0"); - builder.AppendLine("#The address to use for mining (empty string to select an address from the wallet)."); - builder.AppendLine("#mineaddress="); - builder.AppendLine("#The wallet name to use when staking."); - builder.AppendLine("#walletname="); - builder.AppendLine("#Password to unlock the wallet."); - builder.AppendLine("#walletpassword="); - builder.AppendLine("#Maximum block size (in bytes) for the miner to generate."); - builder.AppendLine($"#blockmaxsize={network.Consensus.Options.MaxBlockSerializedSize}"); - builder.AppendLine("#Maximum block weight (in weight units) for the miner to generate."); - builder.AppendLine($"#blockmaxweight={network.Consensus.Options.MaxBlockWeight}"); - builder.AppendLine("#Enable splitting coins when staking."); - builder.AppendLine("#enablecoinstakesplitting=1"); - builder.AppendLine("#Minimum size of the coins considered for staking, in satoshis."); - builder.AppendLine($"#minimumstakingcoinvalue={MinimumStakingCoinValueDefaultValue}"); - builder.AppendLine("#Targeted minimum value of staking coins after splitting, in satoshis."); - builder.AppendLine($"#minimumsplitcoinvalue={MinimumSplitCoinValueDefaultValue}"); } } } diff --git a/src/Stratis.Bitcoin.Features.Miner/MiningFeature.cs b/src/Stratis.Bitcoin.Features.Miner/MiningFeature.cs index b9d309e701..18989e8cc2 100644 --- a/src/Stratis.Bitcoin.Features.Miner/MiningFeature.cs +++ b/src/Stratis.Bitcoin.Features.Miner/MiningFeature.cs @@ -79,7 +79,7 @@ public MiningFeature( /// The network to extract values from. public static void PrintHelp(Network network) { - MinerSettings.PrintHelp(network); + BaseSettings.PrintHelp(typeof(MinerSettings), network); } /// @@ -89,7 +89,7 @@ public static void PrintHelp(Network network) /// The network to base the defaults off. public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) { - MinerSettings.BuildDefaultConfigurationFile(builder, network); + BaseSettings.BuildDefaultConfigurationFile(typeof(MinerSettings), builder, network); } /// diff --git a/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs index 04a09963a3..8ba4733d26 100644 --- a/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs +++ b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs @@ -37,7 +37,7 @@ private static string TypeDescription(Type type) if (type == typeof(bool)) return "0 or 1"; - if (type == typeof(int)) + if (type.IsValueType) return "number"; return type.ToString().Split('.').Last().ToLower(); @@ -66,7 +66,9 @@ public BaseSettings(NodeSettings nodeSettings) TextFileConfiguration config = nodeSettings.ConfigReader; - foreach (PropertyInfo pi in this.GetType().GetProperties()) + BindingFlags bindingFlags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public; + + foreach (PropertyInfo pi in this.GetType().GetProperties(bindingFlags)) { CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); if (attr == null) @@ -80,7 +82,9 @@ public BaseSettings(NodeSettings nodeSettings) typeGetter[pi.PropertyType] = getOrDefault; } - pi.SetValue(this, getOrDefault.Invoke(config, new object[] { attr.Option, pi.GetValue(this), attr.CanLog ? logger : null })); + object? value = getOrDefault.Invoke(config, bindingFlags, null, new object[] { attr.Option, pi.GetValue(this), attr.CanLog ? logger : null }, null); + + pi.SetValue(this, value); } } } diff --git a/src/Stratis.Bitcoin/Configuration/TextFileConfiguration.cs b/src/Stratis.Bitcoin/Configuration/TextFileConfiguration.cs index 3087970c80..9df1006579 100644 --- a/src/Stratis.Bitcoin/Configuration/TextFileConfiguration.cs +++ b/src/Stratis.Bitcoin/Configuration/TextFileConfiguration.cs @@ -204,6 +204,11 @@ private T ConvertValue(string str) return (T)(object)int.Parse(str, CultureInfo.InvariantCulture); } + if (typeof(T) == typeof(uint)) + { + return (T)(object)uint.Parse(str, CultureInfo.InvariantCulture); + } + if (typeof(T) == typeof(long)) { return (T)(object)long.Parse(str, CultureInfo.InvariantCulture); From 696bb0d2ecdf59216c21ff1153e9caacf09a375a Mon Sep 17 00:00:00 2001 From: quantumagi Date: Sat, 4 Jun 2022 10:25:35 +1000 Subject: [PATCH 13/44] Refactor ApiSettings (#993) * Refactor ApiSettings * Add comment * Fix comment * Fix test * Fix tests * Simplify (cherry picked from commit 4d308a9923ec49100eb5f12dfeb14ed6ae48916c) --- .../ApiFeature.cs | 5 +- .../ApiSettings.cs | 108 ++++-------------- .../Configuration/Settings/BaseSettings.cs | 8 +- .../Unity3dApiFeature.cs | 5 +- 4 files changed, 32 insertions(+), 94 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Api/ApiFeature.cs b/src/Stratis.Bitcoin.Features.Api/ApiFeature.cs index 32716123c9..89b01c7ba2 100644 --- a/src/Stratis.Bitcoin.Features.Api/ApiFeature.cs +++ b/src/Stratis.Bitcoin.Features.Api/ApiFeature.cs @@ -7,6 +7,7 @@ using NBitcoin; using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Builder.Feature; +using Stratis.Bitcoin.Configuration.Settings; namespace Stratis.Bitcoin.Features.Api { @@ -83,7 +84,7 @@ public override Task InitializeAsync() /// The network to extract values from. public static void PrintHelp(Network network) { - ApiSettings.PrintHelp(network); + BaseSettings.PrintHelp(typeof(ApiSettings), network); } /// @@ -93,7 +94,7 @@ public static void PrintHelp(Network network) /// The network to base the defaults off. public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) { - ApiSettings.BuildDefaultConfigurationFile(builder, network); + BaseSettings.BuildDefaultConfigurationFile(typeof(ApiSettings), builder, network); } /// diff --git a/src/Stratis.Bitcoin.Features.Api/ApiSettings.cs b/src/Stratis.Bitcoin.Features.Api/ApiSettings.cs index 33e3e0be9f..849c3f239e 100644 --- a/src/Stratis.Bitcoin.Features.Api/ApiSettings.cs +++ b/src/Stratis.Bitcoin.Features.Api/ApiSettings.cs @@ -1,31 +1,35 @@ using System; -using System.Text; using System.Timers; -using Microsoft.Extensions.Logging; -using NBitcoin; using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; +using Stratis.Bitcoin.Configuration.Settings; namespace Stratis.Bitcoin.Features.Api { /// /// Configuration related to the API interface. /// - public class ApiSettings + public class ApiSettings : BaseSettings { /// The default port used by the API when the node runs on the Stratis network. public const string DefaultApiHost = "http://localhost"; - /// Instance logger. - private readonly ILogger logger; - /// URI to node's API interface. - public Uri ApiUri { get; set; } + [CommandLineOption("apiuri", "URI to node's API interface.")] + private string ApiHost { get { return this.UseHttps ? this.apiHost.Replace(@"http://", @"https://") : this.apiHost; } set { this.apiHost = value; } } + private string apiHost = DefaultApiHost; + + // If a port is set in the -apiuri, it takes precedence over the default port or the port passed in -apiport. + public Uri ApiUri { get { Uri uri = new Uri(this.ApiHost); return uri.IsDefaultPort ? new Uri($"{this.ApiHost}:{this.apiPort ?? this.nodeSettings.Network.DefaultAPIPort}") : uri; } } /// Port of node's API interface. - public int ApiPort { get; set; } + [CommandLineOption("apiport", "Port of node's API interface.")] + public int ApiPort { get { return this.ApiUri.Port; } set { this.apiPort = value; } } + private int? apiPort = null; + + /// Sets the keepalive interval (set in seconds). + [CommandLineOption("keepalive", "Keep Alive interval (set in seconds).")] + private int KeepAlive { get; set; } = 0; - /// URI to node's API interface. public Timer KeepaliveTimer { get; private set; } /// @@ -35,99 +39,31 @@ public class ApiSettings /// Password protected certificates are not supported. On MacOs, only p12 certificates can be used without password. /// Please refer to .Net Core documentation for usage: . /// - public string HttpsCertificateFilePath { get; set; } + [CommandLineOption("certificatefilepath", "Path to the certificate used for https traffic encryption. Password protected files are not supported. On MacOs, only p12 certificates can be used without password.", false)] + public string HttpsCertificateFilePath { get; set; } = null; /// Use HTTPS or not. - public bool UseHttps { get; set; } + [CommandLineOption("usehttps", "Use https protocol on the API.")] + public bool UseHttps { get; set; } = false; /// /// Initializes an instance of the object from the node configuration. /// /// The node configuration. - public ApiSettings(NodeSettings nodeSettings) + public ApiSettings(NodeSettings nodeSettings) : base(nodeSettings) { - Guard.NotNull(nodeSettings, nameof(nodeSettings)); - - this.logger = nodeSettings.LoggerFactory.CreateLogger(typeof(ApiSettings).FullName); - - TextFileConfiguration config = nodeSettings.ConfigReader; - - this.UseHttps = config.GetOrDefault("usehttps", false); - this.HttpsCertificateFilePath = config.GetOrDefault("certificatefilepath", (string)null); - if (this.UseHttps && string.IsNullOrWhiteSpace(this.HttpsCertificateFilePath)) throw new ConfigurationException("The path to a certificate needs to be provided when using https. Please use the argument 'certificatefilepath' to provide it."); - var defaultApiHost = this.UseHttps - ? DefaultApiHost.Replace(@"http://", @"https://") - : DefaultApiHost; - - string apiHost = config.GetOrDefault("apiuri", defaultApiHost, this.logger); - var apiUri = new Uri(apiHost); - - // Find out which port should be used for the API. - int apiPort = config.GetOrDefault("apiport", nodeSettings.Network.DefaultAPIPort, this.logger); - - // If no port is set in the API URI. - if (apiUri.IsDefaultPort) - { - this.ApiUri = new Uri($"{apiHost}:{apiPort}"); - this.ApiPort = apiPort; - } - // If a port is set in the -apiuri, it takes precedence over the default port or the port passed in -apiport. - else - { - this.ApiUri = apiUri; - this.ApiPort = apiUri.Port; - } - // Set the keepalive interval (set in seconds). - int keepAlive = config.GetOrDefault("keepalive", 0, this.logger); - if (keepAlive > 0) + if (this.KeepAlive > 0) { this.KeepaliveTimer = new Timer { AutoReset = false, - Interval = keepAlive * 1000 + Interval = this.KeepAlive * 1000 }; } } - - /// Prints the help information on how to configure the API settings to the logger. - /// The network to use. - public static void PrintHelp(Network network) - { - var builder = new StringBuilder(); - - builder.AppendLine($"-apiuri= URI to node's API interface. Defaults to '{ DefaultApiHost }'."); - builder.AppendLine($"-apiport=<0-65535> Port of node's API interface. Defaults to { network.DefaultAPIPort }."); - builder.AppendLine($"-keepalive= Keep Alive interval (set in seconds). Default: 0 (no keep alive)."); - builder.AppendLine($"-usehttps= Use https protocol on the API. Defaults to false."); - builder.AppendLine($"-certificatefilepath= Path to the certificate used for https traffic encryption. Defaults to . Password protected files are not supported. On MacOs, only p12 certificates can be used without password."); - - var logger = NodeSettings.Default(network).LoggerFactory.CreateLogger(typeof(ApiSettings).FullName); - logger.LogInformation(builder.ToString()); - } - - /// - /// Get the default configuration. - /// - /// The string builder to add the settings to. - /// The network to base the defaults off. - public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) - { - builder.AppendLine("####API Settings####"); - builder.AppendLine($"#URI to node's API interface. Defaults to '{ DefaultApiHost }'."); - builder.AppendLine($"#apiuri={ DefaultApiHost }"); - builder.AppendLine($"#Port of node's API interface. Defaults to { network.DefaultAPIPort }."); - builder.AppendLine($"#apiport={ network.DefaultAPIPort }"); - builder.AppendLine($"#Keep Alive interval (set in seconds). Default: 0 (no keep alive)."); - builder.AppendLine($"#keepalive=0"); - builder.AppendLine($"#Use HTTPS protocol on the API. Default is false."); - builder.AppendLine($"#usehttps=false"); - builder.AppendLine($"#Path to the file containing the certificate to use for https traffic encryption. Password protected files are not supported. On MacOs, only p12 certificates can be used without password."); - builder.AppendLine(@"#Please refer to .Net Core documentation for usage: 'https://docs.microsoft.com/en-us/dotnet/api/system.security.cryptography.x509certificates.x509certificate2.-ctor?view=netcore-2.1#System_Security_Cryptography_X509Certificates_X509Certificate2__ctor_System_Byte___'."); - builder.AppendLine($"#certificatefilepath="); - } } } diff --git a/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs index 8ba4733d26..d5ab601807 100644 --- a/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs +++ b/src/Stratis.Bitcoin/Configuration/Settings/BaseSettings.cs @@ -56,6 +56,8 @@ private static string DefaultValue(object value, bool raw = false) private static Dictionary typeGetter = new Dictionary(); + private static BindingFlags bindingFlags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public; + public BaseSettings(NodeSettings nodeSettings) { Guard.NotNull(nodeSettings, nameof(nodeSettings)); @@ -66,8 +68,6 @@ public BaseSettings(NodeSettings nodeSettings) TextFileConfiguration config = nodeSettings.ConfigReader; - BindingFlags bindingFlags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public; - foreach (PropertyInfo pi in this.GetType().GetProperties(bindingFlags)) { CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); @@ -100,7 +100,7 @@ public static void PrintHelp(Type settingsType, Network network) var builder = new StringBuilder(); var defaultValues = Activator.CreateInstance(settingsType, new object[] { defaultSettings }); - foreach (PropertyInfo pi in settingsType.GetProperties()) + foreach (PropertyInfo pi in settingsType.GetProperties(bindingFlags)) { CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); if (attr == null) @@ -130,7 +130,7 @@ public static void BuildDefaultConfigurationFile(Type settingsType, StringBuilde NodeSettings defaultSettings = NodeSettings.Default(network); var defaultValues = Activator.CreateInstance(settingsType, new object[] { defaultSettings }); - foreach (PropertyInfo pi in settingsType.GetProperties()) + foreach (PropertyInfo pi in settingsType.GetProperties(bindingFlags)) { CommandLineOptionAttribute attr = Attribute.GetCustomAttributes(pi).OfType().FirstOrDefault(); if (attr == null) diff --git a/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs b/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs index 52ebf86a1e..10c436e6d2 100644 --- a/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs +++ b/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs @@ -8,6 +8,7 @@ using Stratis.Bitcoin; using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Builder.Feature; +using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Features.Api; using Stratis.Bitcoin.Features.Wallet.Controllers; using Stratis.Features.Unity3dApi.Controllers; @@ -95,7 +96,7 @@ public override Task InitializeAsync() /// The network to extract values from. public static void PrintHelp(Network network) { - ApiSettings.PrintHelp(network); + BaseSettings.PrintHelp(typeof(ApiSettings), network); } /// @@ -105,7 +106,7 @@ public static void PrintHelp(Network network) /// The network to base the defaults off. public static void BuildDefaultConfigurationFile(StringBuilder builder, Network network) { - ApiSettings.BuildDefaultConfigurationFile(builder, network); + BaseSettings.BuildDefaultConfigurationFile(typeof(ApiSettings), builder, network); } /// From 47ec7c31e06fca7f566a69c8ef8e251c1b6c3f19 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 30 Jun 2022 17:57:30 +1000 Subject: [PATCH 14/44] Make BatchBlocksFrom re-usable (#1010) (cherry picked from commit 5226cfe41d9b0c555a14fa703d65100ed3858088) --- .../WalletSyncManager.cs | 36 +----------------- src/Stratis.Bitcoin/Interfaces/IBlockStore.cs | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletSyncManager.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletSyncManager.cs index fef4a435cb..fb3636ceb5 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletSyncManager.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletSyncManager.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -149,7 +148,7 @@ public virtual void ProcessTransaction(Transaction transaction) private void ProcessBlocks() { - this.walletManager.ProcessBlocks((previousBlock) => { return this.BatchBlocksFrom(previousBlock); }); + this.walletManager.ProcessBlocks((previousBlock) => { return this.blockStore.BatchBlocksFrom(previousBlock, this.chainIndexer, this.syncCancellationToken); }); } private void OnBlockConnected(BlockConnected blockConnected) @@ -197,39 +196,6 @@ internal void OrchestrateWalletSync() } } - private IEnumerable<(ChainedHeader, Block)> BatchBlocksFrom(ChainedHeader previousBlock) - { - for (int height = previousBlock.Height + 1; !this.syncCancellationToken.IsCancellationRequested;) - { - var hashes = new List(); - for (int i = 0; i < 100; i++) - { - ChainedHeader header = this.chainIndexer.GetHeader(height + i); - if (header == null) - break; - - if (header.Previous != previousBlock) - break; - - hashes.Add(header.HashBlock); - - previousBlock = header; - } - - if (hashes.Count == 0) - yield break; - - List blocks = this.blockStore.GetBlocks(hashes); - - var buffer = new List<(ChainedHeader, Block)>(); - for (int i = 0; i < blocks.Count && !this.syncCancellationToken.IsCancellationRequested; height++, i++) - { - ChainedHeader header = this.chainIndexer.GetHeader(height); - yield return ((header, blocks[i])); - } - } - } - /// public void Dispose() { diff --git a/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs b/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs index 6f1bc4c238..c0c142364e 100644 --- a/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs +++ b/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs @@ -38,4 +38,42 @@ public interface IBlockStore : IDisposable List GetBlocks(List blockHashes); } + + /// + /// Provides functionality that builds upon the interface. + /// + public static class IBlockStoreExt + { + public static IEnumerable<(ChainedHeader, Block)> BatchBlocksFrom(this IBlockStore blockStore, ChainedHeader previousBlock, ChainIndexer chainIndexer, CancellationTokenSource cancellationToken = null, int batchSize = 100) + { + for (int height = previousBlock.Height + 1; !(cancellationToken?.IsCancellationRequested ?? false);) + { + var hashes = new List(); + for (int i = 0; i < batchSize; i++) + { + ChainedHeader header = chainIndexer.GetHeader(height + i); + if (header == null) + break; + + if (header.Previous != previousBlock) + break; + + hashes.Add(header.HashBlock); + + previousBlock = header; + } + + if (hashes.Count == 0) + yield break; + + List blocks = blockStore.GetBlocks(hashes); + + for (int i = 0; i < blocks.Count && !(cancellationToken?.IsCancellationRequested ?? false); height++, i++) + { + ChainedHeader header = chainIndexer.GetHeader(height); + yield return ((header, blocks[i])); + } + } + } + } } From 580120c29bbe2ef10484601ccb42389c386e6095 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Sat, 2 Jul 2022 18:22:29 +1000 Subject: [PATCH 15/44] Remove ScriptDestinationReader (#1011) * Remove ScriptDestinationReader * Fix tests * Refactor * Add xml comment * Fix * Add serviceLifetime argument * Require explicit lifetime * Simplify * Uncomment code * Match replacement behavior of other Replace methods * Refactor (cherry picked from commit 20e9c3ff6bc3293428bc63b2d9096d0c566b4c07) --- src/NBitcoin/Script.cs | 2 +- src/NBitcoin/ScriptEvaluationContext.cs | 4 +- src/NBitcoin/StandardScriptTemplate.cs | 4 +- .../Controllers/BlockStoreController.cs | 1 + .../ColdStakingDestinationReader.cs | 35 +++++++---- .../ColdStakingFeature.cs | 3 +- .../Rules/CommonRules/CoinviewRule.cs | 2 +- .../Rules/CreateMempoolEntryMempoolRule.cs | 2 +- .../SmartContractScriptAddressReader.cs | 25 ++++++-- .../Wallet/SmartContractWalletFeature.cs | 8 +-- .../Interfaces/ScriptDestinationReader.cs | 50 --------------- .../WalletFeature.cs | 2 +- .../Consensus/ScriptAddressReader.cs | 61 ++++++++++++++----- .../Interfaces/IScriptAddressReader.cs | 46 ++++++++++++-- .../WalletRepositoryTests.cs | 22 +------ .../External/ITransactionsToLists.cs | 49 +-------------- .../SQLiteWalletRepository.cs | 1 + 17 files changed, 150 insertions(+), 167 deletions(-) delete mode 100644 src/Stratis.Bitcoin.Features.Wallet/Interfaces/ScriptDestinationReader.cs diff --git a/src/NBitcoin/Script.cs b/src/NBitcoin/Script.cs index 2c36047682..48abe9f119 100644 --- a/src/NBitcoin/Script.cs +++ b/src/NBitcoin/Script.cs @@ -1021,7 +1021,7 @@ public TxDestination GetDestination(Network network) ScriptId scriptHashParams = PayToScriptHashTemplate.Instance.ExtractScriptPubKeyParameters(this); if (scriptHashParams != null) return scriptHashParams; - TxDestination wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters(network, this); + TxDestination wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters(this); return wit; } diff --git a/src/NBitcoin/ScriptEvaluationContext.cs b/src/NBitcoin/ScriptEvaluationContext.cs index 75937018eb..4d22ef6186 100644 --- a/src/NBitcoin/ScriptEvaluationContext.cs +++ b/src/NBitcoin/ScriptEvaluationContext.cs @@ -463,7 +463,7 @@ public bool VerifyScript(Script scriptSig, Script scriptPubKey, TransactionCheck if((this.ScriptVerify & ScriptVerify.Witness) != 0) { - WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(this.Network, scriptPubKey); + WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(scriptPubKey); if(wit != null) { hadWitness = true; @@ -508,7 +508,7 @@ public bool VerifyScript(Script scriptSig, Script scriptPubKey, TransactionCheck // P2SH witness program if((this.ScriptVerify & ScriptVerify.Witness) != 0) { - WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(this.Network, redeem); + WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(redeem); if(wit != null) { hadWitness = true; diff --git a/src/NBitcoin/StandardScriptTemplate.cs b/src/NBitcoin/StandardScriptTemplate.cs index 2de6166edb..139be1ae3f 100644 --- a/src/NBitcoin/StandardScriptTemplate.cs +++ b/src/NBitcoin/StandardScriptTemplate.cs @@ -1142,7 +1142,7 @@ public static bool ValidSegwitVersion(byte version) return version == 0 || ((byte)OpcodeType.OP_1 <= version && version <= (byte)OpcodeType.OP_16); } - public TxDestination ExtractScriptPubKeyParameters(Network network, Script scriptPubKey) + public TxDestination ExtractScriptPubKeyParameters(Script scriptPubKey) { if(!CheckScriptPubKey(scriptPubKey)) return null; @@ -1158,7 +1158,7 @@ public TxDestination ExtractScriptPubKeyParameters(Network network, Script scrip } return null; } - public WitProgramParameters ExtractScriptPubKeyParameters2(Network network, Script scriptPubKey) + public WitProgramParameters ExtractScriptPubKeyParameters2(Script scriptPubKey) { if(!CheckScriptPubKey(scriptPubKey)) return null; diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs index 8573881532..0e8098dec8 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Base; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Controllers.Models; using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.BlockStore.Models; diff --git a/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingDestinationReader.cs b/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingDestinationReader.cs index dee9cc9c56..61e660cff8 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingDestinationReader.cs +++ b/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingDestinationReader.cs @@ -1,31 +1,42 @@ using System.Collections.Generic; using NBitcoin; using Stratis.Bitcoin.Consensus; -using Stratis.Bitcoin.Features.Wallet.Interfaces; +using Stratis.Bitcoin.Interfaces; namespace Stratis.Bitcoin.Features.ColdStaking { /// - /// This class and the base offers the ability to selectively replace + /// This class offers the ability to selectively replace /// which can only parse a single address from a ScriptPubKey. ColdStaking scripts contain two addresses / pub key hashes. /// - public class ColdStakingDestinationReader : ScriptDestinationReader, IScriptDestinationReader + public class ColdStakingDestinationReader : IScriptAddressReader { - public ColdStakingDestinationReader(ScriptAddressReader scriptAddressReader) : base(scriptAddressReader) + IScriptAddressReader scriptAddressReader; + + public ColdStakingDestinationReader(IScriptAddressReader scriptAddressReader) { + this.scriptAddressReader = scriptAddressReader ?? new ScriptAddressReader(); } - public override IEnumerable GetDestinationFromScriptPubKey(Network network, Script redeemScript) + public string GetAddressFromScriptPubKey(ScriptTemplate scriptTemplate, Network network, Script script) { - if (ColdStakingScriptTemplate.Instance.ExtractScriptPubKeyParameters(redeemScript, out KeyId hotPubKeyHash, out KeyId coldPubKeyHash)) - { - yield return hotPubKeyHash; - yield return coldPubKeyHash; - } - else + return this.scriptAddressReader.GetAddressFromScriptPubKey(scriptTemplate, network, script); + } + + public IEnumerable GetDestinationFromScriptPubKey(ScriptTemplate scriptTemplate, Script script) + { + if (scriptTemplate.Type == TxOutType.TX_COLDSTAKE && ((ColdStakingScriptTemplate)scriptTemplate).ExtractScriptPubKeyParameters(script, out KeyId hotPubKeyHash, out KeyId coldPubKeyHash)) { - base.GetDestinationFromScriptPubKey(network, redeemScript); + IEnumerable GetDestinations() + { + yield return hotPubKeyHash; + yield return coldPubKeyHash; + } + + return GetDestinations(); } + + return this.scriptAddressReader.GetDestinationFromScriptPubKey(scriptTemplate, script); } } } diff --git a/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingFeature.cs b/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingFeature.cs index ff0084a74c..6e42c00446 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingFeature.cs +++ b/src/Stratis.Bitcoin.Features.ColdStaking/ColdStakingFeature.cs @@ -245,8 +245,7 @@ public static IFullNodeBuilder UseColdStakingWallet(this IFullNodeBuilder fullNo services.RemoveAll(); services.AddSingleton(); - services.AddSingleton(); - services.Replace(new ServiceDescriptor(typeof(IScriptAddressReader), typeof(ColdStakingDestinationReader), ServiceLifetime.Singleton)); + services.Replace((p, old) => new ColdStakingDestinationReader(old), ServiceLifetime.Singleton); }); }); diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/CoinviewRule.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/CoinviewRule.cs index 05c8617d84..ad280c3228 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/CoinviewRule.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/CoinviewRule.cs @@ -361,7 +361,7 @@ private long CountWitnessSignatureOperation(Script scriptPubKey, WitScript witne if (!flags.ScriptFlags.HasFlag(ScriptVerify.Witness)) return 0; - WitProgramParameters witParams = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(this.Parent.Network, scriptPubKey); + WitProgramParameters witParams = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(scriptPubKey); if (witParams?.Version == 0) { diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Rules/CreateMempoolEntryMempoolRule.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Rules/CreateMempoolEntryMempoolRule.cs index 2acf1e0952..023e91e389 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Rules/CreateMempoolEntryMempoolRule.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Rules/CreateMempoolEntryMempoolRule.cs @@ -294,7 +294,7 @@ private bool IsWitnessStandard(Transaction tx, MempoolCoinView mapInputs) } // Check P2WSH standard limits. - WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(this.chainIndexer.Network, prevScript); + WitProgramParameters wit = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters2(prevScript); if (wit == null) { this.logger.LogTrace("(-)[BAD_WITNESS_PARAMS]:false"); diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/SmartContractScriptAddressReader.cs b/src/Stratis.Bitcoin.Features.SmartContracts/SmartContractScriptAddressReader.cs index dfbf91b221..de22641677 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/SmartContractScriptAddressReader.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/SmartContractScriptAddressReader.cs @@ -1,4 +1,5 @@ -using CSharpFunctionalExtensions; +using System.Collections.Generic; +using CSharpFunctionalExtensions; using NBitcoin; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Interfaces; @@ -15,14 +16,14 @@ public sealed class SmartContractScriptAddressReader : IScriptAddressReader private readonly ICallDataSerializer callDataSerializer; public SmartContractScriptAddressReader( - ScriptAddressReader addressReader, + IScriptAddressReader addressReader, ICallDataSerializer callDataSerializer) { - this.baseAddressReader = addressReader; + this.baseAddressReader = addressReader ?? new ScriptAddressReader(); this.callDataSerializer = callDataSerializer; } - public string GetAddressFromScriptPubKey(Network network, Script script) + public string GetAddressFromScriptPubKey(ScriptTemplate scriptTemplate, Network network, Script script) { if (script.IsSmartContractCreate() || script.IsSmartContractCall()) { @@ -32,5 +33,21 @@ public string GetAddressFromScriptPubKey(Network network, Script script) return this.baseAddressReader.GetAddressFromScriptPubKey(network, script); } + + public IEnumerable GetDestinationFromScriptPubKey(ScriptTemplate scriptTemplate, Script script) + { + if (script.IsSmartContractCreate() || script.IsSmartContractCall()) + { + IEnumerable Destinations() + { + Result result = this.callDataSerializer.Deserialize(script.ToBytes()); + yield return new KeyId(result.Value.ContractAddress); + } + + return Destinations(); + } + + return this.baseAddressReader.GetDestinationFromScriptPubKey(scriptTemplate, script); + } } } \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/Wallet/SmartContractWalletFeature.cs b/src/Stratis.Bitcoin.Features.SmartContracts/Wallet/SmartContractWalletFeature.cs index f7bd563423..8e7853acd2 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/Wallet/SmartContractWalletFeature.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/Wallet/SmartContractWalletFeature.cs @@ -6,10 +6,10 @@ using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Builder.Feature; using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Wallet; using Stratis.Bitcoin.Features.Wallet.Interfaces; using Stratis.Bitcoin.Interfaces; +using Stratis.SmartContracts.CLR; namespace Stratis.Bitcoin.Features.SmartContracts.Wallet { @@ -54,10 +54,8 @@ public static IFullNodeBuilder UseSmartContractWallet(this IFullNodeBuilder full .DependOn() .FeatureServices(services => { - // Registers the ScriptAddressReader concrete type and replaces the IScriptAddressReader implementation - // with SmartContractScriptAddressReader, which depends on the ScriptAddressReader concrete type. - services.AddSingleton(); - services.Replace(new ServiceDescriptor(typeof(IScriptAddressReader), typeof(SmartContractScriptAddressReader), ServiceLifetime.Singleton)); + // Replaces the IScriptAddressReader implementation with SmartContractScriptAddressReader, chaining to the old service implementation. + services.Replace((p, old) => new SmartContractScriptAddressReader(old, p.GetService()), ServiceLifetime.Singleton); services.RemoveAll(typeof(StandardTransactionPolicy)); services.AddSingleton(); diff --git a/src/Stratis.Bitcoin.Features.Wallet/Interfaces/ScriptDestinationReader.cs b/src/Stratis.Bitcoin.Features.Wallet/Interfaces/ScriptDestinationReader.cs deleted file mode 100644 index bafb1e05f5..0000000000 --- a/src/Stratis.Bitcoin.Features.Wallet/Interfaces/ScriptDestinationReader.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using NBitcoin; -using NBitcoin.DataEncoders; -using Stratis.Bitcoin.Interfaces; - -namespace Stratis.Bitcoin.Features.Wallet.Interfaces -{ - public interface IScriptDestinationReader : IScriptAddressReader - { - IEnumerable GetDestinationFromScriptPubKey(Network network, Script script); - } - - public class ScriptDestinationReader : IScriptAddressReader - { - private readonly IScriptAddressReader scriptAddressReader; - - public ScriptDestinationReader(IScriptAddressReader scriptAddressReader) - { - this.scriptAddressReader = scriptAddressReader; - } - - public string GetAddressFromScriptPubKey(Network network, Script script) - { - return this.scriptAddressReader.GetAddressFromScriptPubKey(network, script); - } - - public virtual IEnumerable GetDestinationFromScriptPubKey(Network network, Script redeemScript) - { - string base58 = this.scriptAddressReader.GetAddressFromScriptPubKey(network, redeemScript); - - if (base58 != null) - { - TxDestination destination = ScriptDestinationReader.GetDestinationForAddress(base58, network); - - if (destination != null) - yield return destination; - } - } - - public static TxDestination GetDestinationForAddress(string address, Network network) - { - if (address == null) - return null; - - byte[] decoded = Encoders.Base58Check.DecodeData(address); - return new KeyId(new uint160(decoded.Skip(network.GetVersionBytes(Base58Type.PUBKEY_ADDRESS, true).Length).ToArray())); - } - } -} diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs index d539f9d203..2c518af7ff 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletFeature.cs @@ -182,7 +182,7 @@ public static IFullNodeBuilder UseWallet(this IFullNodeBuilder fullNodeBuilder) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(new ScriptAddressReader()); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Stratis.Bitcoin/Consensus/ScriptAddressReader.cs b/src/Stratis.Bitcoin/Consensus/ScriptAddressReader.cs index aadc0f526f..f9162803eb 100644 --- a/src/Stratis.Bitcoin/Consensus/ScriptAddressReader.cs +++ b/src/Stratis.Bitcoin/Consensus/ScriptAddressReader.cs @@ -1,19 +1,39 @@ -using NBitcoin; +using System.Collections.Generic; +using NBitcoin; using Stratis.Bitcoin.Interfaces; namespace Stratis.Bitcoin.Consensus { + public static class ScriptAddressReaderExt + { + public static string GetAddressFromScriptPubKey(this IScriptAddressReader scriptAddressReader, Network network, Script script) + { + ScriptTemplate scriptTemplate = network.StandardScriptsRegistry.GetTemplateFromScriptPubKey(script); + if (scriptTemplate == null) + return null; + + return scriptAddressReader.GetAddressFromScriptPubKey(scriptTemplate, network, script); + } + + public static IEnumerable GetDestinationFromScriptPubKey(this IScriptAddressReader scriptAddressReader, Network network, Script script) + { + ScriptTemplate scriptTemplate = network.StandardScriptsRegistry.GetTemplateFromScriptPubKey(script); + if (scriptTemplate == null) + return new TxDestination[0]; + + return scriptAddressReader.GetDestinationFromScriptPubKey(scriptTemplate, script); + } + } + /// public class ScriptAddressReader : IScriptAddressReader { /// - public string GetAddressFromScriptPubKey(Network network, Script script) + public string GetAddressFromScriptPubKey(ScriptTemplate scriptTemplate, Network network, Script script) { - ScriptTemplate scriptTemplate = network.StandardScriptsRegistry.GetTemplateFromScriptPubKey(script); - string destinationAddress = null; - switch (scriptTemplate?.Type) + switch (scriptTemplate.Type) { // Pay to PubKey can be found in outputs of staking transactions. case TxOutType.TX_PUBKEY: @@ -22,23 +42,34 @@ public string GetAddressFromScriptPubKey(Network network, Script script) break; // Pay to PubKey hash is the regular, most common type of output. case TxOutType.TX_PUBKEYHASH: - destinationAddress = script.GetDestinationAddress(network).ToString(); - break; case TxOutType.TX_SCRIPTHASH: - destinationAddress = script.GetDestinationAddress(network).ToString(); - break; case TxOutType.TX_SEGWIT: destinationAddress = script.GetDestinationAddress(network).ToString(); break; - case TxOutType.TX_NONSTANDARD: - case TxOutType.TX_MULTISIG: - case TxOutType.TX_NULL_DATA: - case TxOutType.TX_COLDSTAKE: - case TxOutType.TX_FEDERATION: - break; } return destinationAddress; } + + public IEnumerable GetDestinationFromScriptPubKey(ScriptTemplate scriptTemplate, Script script) + { + switch (scriptTemplate.Type) + { + case TxOutType.TX_PUBKEYHASH: + yield return PayToPubkeyHashTemplate.Instance.ExtractScriptPubKeyParameters(script); + break; + case TxOutType.TX_PUBKEY: + yield return PayToPubkeyTemplate.Instance.ExtractScriptPubKeyParameters(script).Hash; + break; + case TxOutType.TX_SCRIPTHASH: + yield return PayToScriptHashTemplate.Instance.ExtractScriptPubKeyParameters(script); + break; + case TxOutType.TX_SEGWIT: + TxDestination txDestination = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters(script); + if (txDestination != null) + yield return new KeyId(txDestination.ToBytes()); + break; + } + } } } diff --git a/src/Stratis.Bitcoin/Interfaces/IScriptAddressReader.cs b/src/Stratis.Bitcoin/Interfaces/IScriptAddressReader.cs index 8a6c3aebe0..cf6db06cb8 100644 --- a/src/Stratis.Bitcoin/Interfaces/IScriptAddressReader.cs +++ b/src/Stratis.Bitcoin/Interfaces/IScriptAddressReader.cs @@ -1,4 +1,9 @@ -using NBitcoin; +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using NBitcoin; namespace Stratis.Bitcoin.Interfaces { @@ -10,9 +15,42 @@ public interface IScriptAddressReader /// /// Extracts an address from a given Script, if available. Otherwise returns /// - /// - /// + /// The appropriate template for this type of script. + /// The network. + /// The script. /// - string GetAddressFromScriptPubKey(Network network, Script script); + string GetAddressFromScriptPubKey(ScriptTemplate scriptTemplate, Network network, Script script); + + IEnumerable GetDestinationFromScriptPubKey(ScriptTemplate scriptTemplate, Script script); + } + + public static class ServiceDescriptorExt + { + public static T MakeConcrete(this ServiceDescriptor service, IServiceProvider provider) + { + return (T)(service.ImplementationInstance ?? service.ImplementationFactory?.Invoke(provider) ?? ActivatorUtilities.CreateInstance(provider, service.ImplementationType)); + } + } + + public static class IServiceCollectionExt + { + /// + /// Replaces a service and provides a factory for creating a new instance that chains to the previous implementation. + /// + /// The service type. + /// The services collection. + /// The factory used to create a new instance that chains to the previous implementation. + /// + public static IServiceCollection Replace(this IServiceCollection services, Func factory, ServiceLifetime serviceLifetime) where I : class + { + ServiceDescriptor previous = services.LastOrDefault(s => s.ServiceType == typeof(I)); + + if (previous != null) + services.Remove(previous); + + services.Add(new ServiceDescriptor(typeof(I), provider => factory(provider, previous?.MakeConcrete(provider)), serviceLifetime)); + + return services; + } } } diff --git a/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs b/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs index b1a8409c91..db0e2cb484 100644 --- a/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs +++ b/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs @@ -52,27 +52,7 @@ public MultiWalletRepositoryTests() : base(false) { } } - - public class ColdStakingDestinationReader : ScriptDestinationReader, IScriptDestinationReader - { - public ColdStakingDestinationReader(IScriptAddressReader scriptAddressReader) : base(scriptAddressReader) - { - } - - public override IEnumerable GetDestinationFromScriptPubKey(Network network, Script redeemScript) - { - if (ColdStakingScriptTemplate.Instance.ExtractScriptPubKeyParameters(redeemScript, out KeyId hotPubKeyHash, out KeyId coldPubKeyHash)) - { - yield return hotPubKeyHash; - yield return coldPubKeyHash; - } - else - { - base.GetDestinationFromScriptPubKey(network, redeemScript); - } - } - } - + public class BlockBase { public NodeSettings NodeSettings { get; private set; } diff --git a/src/Stratis.Features.SQLiteWalletRepository/External/ITransactionsToLists.cs b/src/Stratis.Features.SQLiteWalletRepository/External/ITransactionsToLists.cs index 4213121851..41a02f0a02 100644 --- a/src/Stratis.Features.SQLiteWalletRepository/External/ITransactionsToLists.cs +++ b/src/Stratis.Features.SQLiteWalletRepository/External/ITransactionsToLists.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using NBitcoin; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Wallet; using Stratis.Bitcoin.Features.Wallet.Interfaces; using Stratis.Bitcoin.Interfaces; @@ -34,50 +35,6 @@ public TransactionsToListsBase(Network network, IScriptAddressReader scriptAddre this.dateTimeProvider = dateTimeProvider; } - internal IEnumerable GetDestinations(Script redeemScript) - { - ScriptTemplate scriptTemplate = this.network.StandardScriptsRegistry.GetTemplateFromScriptPubKey(redeemScript); - - if (scriptTemplate != null) - { - // We need scripts suitable for matching to HDAddress.ScriptPubKey. - switch (scriptTemplate.Type) - { - case TxOutType.TX_PUBKEYHASH: - yield return PayToPubkeyHashTemplate.Instance.ExtractScriptPubKeyParameters(redeemScript); - break; - case TxOutType.TX_PUBKEY: - yield return PayToPubkeyTemplate.Instance.ExtractScriptPubKeyParameters(redeemScript).Hash; - break; - case TxOutType.TX_SCRIPTHASH: - yield return PayToScriptHashTemplate.Instance.ExtractScriptPubKeyParameters(redeemScript); - break; - case TxOutType.TX_SEGWIT: - TxDestination txDestination = PayToWitTemplate.Instance.ExtractScriptPubKeyParameters(this.network, redeemScript); - if (txDestination != null) - yield return new KeyId(txDestination.ToBytes()); - break; - default: - if (this.scriptAddressReader is ScriptDestinationReader scriptDestinationReader) - { - foreach (TxDestination destination in scriptDestinationReader.GetDestinationFromScriptPubKey(this.network, redeemScript)) - { - yield return destination; - } - } - else - { - string address = this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, redeemScript); - TxDestination destination = ScriptDestinationReader.GetDestinationForAddress(address, this.network); - if (destination != null) - yield return destination; - } - - break; - } - } - } - public bool ProcessTransactions(IEnumerable transactions, HashHeightPair block, uint256 fixedTxId = null, long? blockTime = null) { bool additions = false; @@ -122,7 +79,7 @@ public bool ProcessTransactions(IEnumerable transactions, HashHeigh if (scriptPubKeyBytes.Length == 0 || scriptPubKeyBytes[0] == (byte)OpcodeType.OP_RETURN) continue; - var destinations = this.GetDestinations(txOut.ScriptPubKey); + var destinations = this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, txOut.ScriptPubKey); bool isChange = destinations.Any(d => addressesOfInterest.Contains(d.ScriptPubKey, out AddressIdentifier address2) && address2.AddressType == 1); @@ -218,7 +175,7 @@ internal void ProcessTransactionData(HdAddress address, ICollection d.ScriptPubKey)) + foreach (Script pubKeyScript in this.scriptAddressReader.GetDestinationFromScriptPubKey(this.network, scriptPubKey).Select(d => d.ScriptPubKey)) { bool containsAddress = addressesOfInterest.Contains(pubKeyScript, out AddressIdentifier targetAddress); diff --git a/src/Stratis.Features.SQLiteWalletRepository/SQLiteWalletRepository.cs b/src/Stratis.Features.SQLiteWalletRepository/SQLiteWalletRepository.cs index 5753fbefab..862e006bd2 100644 --- a/src/Stratis.Features.SQLiteWalletRepository/SQLiteWalletRepository.cs +++ b/src/Stratis.Features.SQLiteWalletRepository/SQLiteWalletRepository.cs @@ -10,6 +10,7 @@ using NBitcoin; using NBitcoin.DataEncoders; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Wallet; using Stratis.Bitcoin.Features.Wallet.Events; using Stratis.Bitcoin.Features.Wallet.Interfaces; From 1b8182b9733b189cbb1580acd6cceb8f54385994 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 4 Jul 2022 23:32:14 +1000 Subject: [PATCH 16/44] Optimise tx hash caclulation (#1015) (cherry picked from commit 957963141b9b94798b4976aada94dc07dae2fc55) --- .../CoinViews/CoinviewHelper.cs | 4 +++- src/Stratis.Bitcoin.Features.Consensus/UnspentOutputSet.cs | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinviewHelper.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinviewHelper.cs index bc3f268f2a..c8b041204e 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinviewHelper.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinviewHelper.cs @@ -20,8 +20,10 @@ public OutPoint[] GetIdsToFetch(Block block, bool enforceBIP30) { if (enforceBIP30) { + // Calculate the hash outside the loop. + var txId = tx.GetHash(); foreach (var utxo in tx.Outputs.AsIndexedOutputs()) - ids.Add(utxo.ToOutPoint()); + ids.Add(new OutPoint() { Hash = txId, N = utxo.N }); } if (!tx.IsCoinBase) diff --git a/src/Stratis.Bitcoin.Features.Consensus/UnspentOutputSet.cs b/src/Stratis.Bitcoin.Features.Consensus/UnspentOutputSet.cs index 41b71318de..2aa58e4ce2 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/UnspentOutputSet.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/UnspentOutputSet.cs @@ -52,9 +52,12 @@ public void Update(Network network, Transaction transaction, int height) } } - foreach (IndexedTxOut output in transaction.Outputs.AsIndexedOutputs()) + // Hash calculations are slow. Do this one outside the loop... + var txHash = transaction.GetHash(); + + foreach (IndexedTxOut output in transaction.Outputs.AsIndexedOutputs().ToList()) { - var outpoint = output.ToOutPoint(); + var outpoint = new OutPoint() { Hash = txHash, N = output.N }; var coinbase = transaction.IsCoinBase; var coinstake = network.Consensus.IsProofOfStake ? transaction.IsCoinStake : false; From 5c2317ebf2d413ebd96822368b78e38910bf71da Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 4 Jul 2022 23:32:38 +1000 Subject: [PATCH 17/44] Move CachedCoinView initialization from PowConsensusRuleEngine (#1014) (cherry picked from commit a12399aa15de8edb72a90981ad3c690df7348286) --- .../CoinViews/CachedCoinView.cs | 24 +++++++++++++++++++ .../CoinViews/CoinView.cs | 7 ++++++ .../CoinViews/InMemoryCoinView.cs | 6 +++++ .../Rules/PowConsensusRuleEngine.cs | 22 +---------------- .../MemPoolCoinView.cs | 5 ++++ .../Consensus/TestInMemoryCoinView.cs | 6 +++++ 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index 890cd8d2f2..be7b38ecd5 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -160,6 +160,30 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300); } + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + { + this.coindb.Initialize(chainTip); + + HashHeightPair coinViewTip = this.coindb.GetTipHash(); + + while (true) + { + ChainedHeader pendingTip = chainTip.FindAncestorOrSelf(coinViewTip.Hash); + + if (pendingTip != null) + break; + + if ((coinViewTip.Height % 100) == 0) + this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, chainTip); + + // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. + // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. + coinViewTip = this.coindb.Rewind(new HashHeightPair(chainTip)); + } + + this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash()); + } + public HashHeightPair GetTipHash() { if (this.blockHash == null) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs index f2b6469361..e0ac6e45a7 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs @@ -10,6 +10,13 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews /// public interface ICoinView { + /// + /// Initializes the coin view. + /// + /// The chain tip. + /// The chain indexer. + void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer); + /// /// Retrieves the block hash of the current tip of the coinview. /// diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 2f2bd59f15..11073a96da 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -32,6 +32,12 @@ public InMemoryCoinView(HashHeightPair tipHash) this.tipHash = tipHash; } + /// + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + { + throw new NotImplementedException(); + } + /// public HashHeightPair GetTipHash() { diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs index 97ade69440..60dd9519b7 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs @@ -67,27 +67,7 @@ public override void Initialize(ChainedHeader chainTip) { base.Initialize(chainTip); - var coinDatabase = ((CachedCoinView)this.UtxoSet).ICoindb; - coinDatabase.Initialize(chainTip); - - HashHeightPair coinViewTip = coinDatabase.GetTipHash(); - - while (true) - { - ChainedHeader pendingTip = chainTip.FindAncestorOrSelf(coinViewTip.Hash); - - if (pendingTip != null) - break; - - if ((coinViewTip.Height % 100) == 0) - this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, chainTip); - - // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. - // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. - coinViewTip = coinDatabase.Rewind(new HashHeightPair(chainTip)); - } - - this.logger.LogInformation("Coin view initialized at '{0}'.", coinDatabase.GetTipHash()); + this.UtxoSet.Initialize(chainTip, this.ChainIndexer); } public override async Task FullValidationAsync(ChainedHeader header, Block block) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs index 91f98a282a..286478d185 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs @@ -49,6 +49,11 @@ public MempoolCoinView(Network network, ICoinView inner, ITxMempool memPool, Sch this.Set = new UnspentOutputSet(); } + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + { + throw new NotImplementedException(); + } + /// /// Gets the unspent transaction output set. /// diff --git a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs index 3ebdd093a1..35fd7898fa 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs @@ -33,6 +33,12 @@ public TestInMemoryCoinView(HashHeightPair tipHash) this.tipHash = tipHash; } + /// + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + { + throw new NotImplementedException(); + } + /// public HashHeightPair GetTipHash() { From 5f69d62c02241562f9a3d1b9beda3253a851a9b3 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Wed, 6 Jul 2022 21:37:52 +1000 Subject: [PATCH 18/44] Optimize PubKey ToHex, GetHashCode and Equals (#968) * Fix merge * Optimize PubKey ToHex, GetHashCode and Equals (cherry picked from commit 91420ac20a57d6c05efe9c08008f37ed0bb0aeeb) --- src/NBitcoin/PubKey.cs | 44 +++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/NBitcoin/PubKey.cs b/src/NBitcoin/PubKey.cs index 06dbb83011..f95a0edcfb 100644 --- a/src/NBitcoin/PubKey.cs +++ b/src/NBitcoin/PubKey.cs @@ -174,9 +174,13 @@ public bool Verify(uint256 hash, byte[] sig) return Verify(hash, ECDSASignature.FromDER(sig)); } + private string hexStr = null; + public string ToHex() { - return Encoders.Hex.EncodeData(this.vch); + this.hexStr ??= Encoders.Hex.EncodeData(this.vch); + + return this.hexStr; } #region IBitcoinSerializable Members @@ -184,7 +188,12 @@ public string ToHex() public void ReadWrite(BitcoinStream stream) { stream.ReadWrite(ref this.vch); - if(!stream.Serializing) this._ECKey = new ECKey(this.vch, false); + + if (!stream.Serializing) + { + this._ECKey = new ECKey(this.vch, false); + this.hexStr = null; + } } #endregion @@ -345,18 +354,33 @@ public PubKey Derivate(byte[] cc, uint nChild, out byte[] ccChild) public override bool Equals(object obj) { - var item = obj as PubKey; - if(item == null) + return obj != null && Equals(obj as PubKey); + } + + public bool Equals(PubKey other) + { + if ((object)other == null) + return false; + + if (other.vch.Length != this.vch.Length) return false; - return ToHex().Equals(item.ToHex()); + + for (int i = 0; i < other.vch.Length; i++) + if (other.vch[i] != this.vch[i]) + return false; + + return true; } + public static bool operator ==(PubKey a, PubKey b) { - if(ReferenceEquals(a, b)) + if (ReferenceEquals(a, b)) return true; - if(((object)a == null) || ((object)b == null)) + + if (((object)a == null) != ((object)b == null)) return false; - return a.ToHex() == b.ToHex(); + + return a.Equals(b); } public static bool operator !=(PubKey a, PubKey b) @@ -366,17 +390,19 @@ public override bool Equals(object obj) public override int GetHashCode() { - return ToHex().GetHashCode(); + return (((((this.vch[1] << 8) + this.vch[2]) << 8) + this.vch[3]) << 8) + this.vch[4]; } public PubKey UncoverSender(Key ephem, PubKey scan) { return Uncover(ephem, scan); } + public PubKey UncoverReceiver(Key scan, PubKey ephem) { return Uncover(scan, ephem); } + public PubKey Uncover(Key priv, PubKey pub) { X9ECParameters curve = ECKey.Secp256k1; From 46166757222c78b8f52c1c6b0161729b4fdaa86a Mon Sep 17 00:00:00 2001 From: quantumagi Date: Wed, 6 Jul 2022 22:13:21 +1000 Subject: [PATCH 19/44] [IBD/Rewind] Optimize P2PKHNotContractRule (#930) * [IBD/Rewind] Optimize P2PKHNotContractRule * Use SkipValidation (cherry picked from commit a40119ea424252739ad6f4551e64e703a9a2169a) --- .../Rules/P2PKHNotContractRule.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/Rules/P2PKHNotContractRule.cs b/src/Stratis.Bitcoin.Features.SmartContracts/Rules/P2PKHNotContractRule.cs index c23c9190cc..38e971d53b 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/Rules/P2PKHNotContractRule.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/Rules/P2PKHNotContractRule.cs @@ -21,6 +21,9 @@ public P2PKHNotContractRule(IStateRepositoryRoot stateRepositoryRoot) /// public override Task RunAsync(RuleContext context) { + if (context.SkipValidation) + return Task.CompletedTask; + Block block = context.ValidationContext.BlockToValidate; foreach (Transaction transaction in block.Transactions) From 31bfe8fea13743d0140f408626709382f45267a1 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 12 Jul 2022 16:38:20 +1000 Subject: [PATCH 20/44] Enable coin view to "catch up" if forked or below chain tip (#1016) * Coin view to catch-up if off-chain or below chain tip * Changes based on feedback * Fix missing argument (cherry picked from commit 6fc76e1a45706bcfd69634e0d5e6b696b7f844f4) --- .../CoinViews/CachedCoinView.cs | 102 +++++++++++++++--- .../CoinViews/CoinView.cs | 10 +- .../CoinViews/InMemoryCoinView.cs | 8 +- .../Rules/CommonRules/LoadCoinviewRule.cs | 3 +- .../Rules/PowConsensusRuleEngine.cs | 2 +- .../MemPoolCoinView.cs | 8 +- .../Consensus/TestInMemoryCoinView.cs | 8 +- 7 files changed, 121 insertions(+), 20 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index be7b38ecd5..e77ad63bb0 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -2,11 +2,14 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; +using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; +using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Utilities; using Stratis.Bitcoin.Utilities.Extensions; using TracerAttributes; @@ -126,13 +129,15 @@ public long GetScriptSize private readonly Network network; private readonly ICheckpoints checkpoints; private readonly IDateTimeProvider dateTimeProvider; + private readonly IBlockStore blockStore; + private readonly CancellationTokenSource cancellationToken; private readonly ConsensusSettings consensusSettings; private CachePerformanceSnapshot latestPerformanceSnapShot; - private int lastCheckpointHeight; private readonly Random random; - public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null) + public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, + StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); @@ -144,6 +149,8 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, this.consensusSettings = consensusSettings; this.stakeChainStore = stakeChainStore; this.rewindDataIndexCache = rewindDataIndexCache; + this.blockStore = blockStore; + this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping); this.lockobj = new object(); this.cachedUtxoItems = new Dictionary(); this.performanceCounter = new CachePerformanceCounter(this.dateTimeProvider); @@ -151,8 +158,6 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, this.cachedRewindData = new Dictionary(); this.random = new Random(); - this.lastCheckpointHeight = this.checkpoints.GetLastCheckpointHeight(); - this.MaxCacheSizeBytes = consensusSettings.MaxCoindbCacheInMB * 1024 * 1024; this.CacheFlushTimeIntervalSeconds = consensusSettings.CoindbIbdFlushMin * 60; @@ -160,25 +165,94 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300); } - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + /// + /// Remain on-chain. + /// + public void Sync(ChainIndexer chainIndexer) + { + lock (this.lockobj) + { + HashHeightPair coinViewTip = this.GetTipHash(); + if (coinViewTip.Hash == chainIndexer.Tip.HashBlock) + return; + + Flush(); + + ChainedHeader fork = chainIndexer[coinViewTip.Hash]; + if (fork == null) + { + // Determine the last usable height. + int height = BinarySearch.BinaryFindFirst(h => (h > chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1; + fork = chainIndexer[(height < 0) ? coinViewTip.Height : height]; + } + + while (coinViewTip.Height != fork.Height) + { + if ((coinViewTip.Height % 100) == 0) + this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork); + + // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. + // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. + coinViewTip = this.coindb.Rewind(new HashHeightPair(fork)); + }; + } + } + + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { this.coindb.Initialize(chainTip); + Sync(chainIndexer); + HashHeightPair coinViewTip = this.coindb.GetTipHash(); - while (true) + // If the coin view is behind the block store then catch up from the block store. + if (coinViewTip.Height < chainTip.Height) { - ChainedHeader pendingTip = chainTip.FindAncestorOrSelf(coinViewTip.Hash); + try + { + var loadCoinViewRule = consensusRuleEngine.GetRule(); + var saveCoinViewRule = consensusRuleEngine.GetRule(); + var coinViewRule = consensusRuleEngine.GetRule(); + var deploymentsRule = consensusRuleEngine.GetRule(); + + foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(chainIndexer[coinViewTip.Hash], chainIndexer, this.cancellationToken, batchSize: 1000)) + { + if (block == null) + break; + + if ((chainedHeader.Height % 10000) == 0) + { + this.Flush(true); + this.logger.LogInformation("Rebuilding coin view from '{0}' to {1}.", chainedHeader, chainTip); + } + + var utxoRuleContext = consensusRuleEngine.CreateRuleContext(new ValidationContext() { ChainedHeaderToValidate = chainedHeader, BlockToValidate = block }); + utxoRuleContext.SkipValidation = true; - if (pendingTip != null) - break; + // Set context flags. + deploymentsRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); - if ((coinViewTip.Height % 100) == 0) - this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, chainTip); + // Loads the coins spent by this block into utxoRuleContext.UnspentOutputSet. + loadCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); - // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. - // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. - coinViewTip = this.coindb.Rewind(new HashHeightPair(chainTip)); + // Spends the coins. + coinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); + + // Saves the changes to the coinview. + saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); + } + } + finally + { + this.Flush(true); + + if (this.cancellationToken.IsCancellationRequested) + { + this.logger.LogDebug("Rebuilding cancelled due to application stopping."); + throw new OperationCanceledException(); + } + } } this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash()); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs index e0ac6e45a7..eca612cdb9 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using NBitcoin; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.CoinViews @@ -15,7 +16,8 @@ public interface ICoinView /// /// The chain tip. /// The chain indexer. - void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer); + /// The consensus rule engine. + void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine); /// /// Retrieves the block hash of the current tip of the coinview. @@ -40,6 +42,12 @@ public interface ICoinView /// List of rewind data items to be persisted. This should only be used when calling . void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null); + /// + /// Brings the coinview back on-chain if a re-org occurred. + /// + /// The current consensus chain. + void Sync(ChainIndexer chainIndexer); + /// /// Obtains information about unspent outputs. /// diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 11073a96da..9cb80e7129 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using NBitcoin; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Utilities; using ReaderWriterLock = NBitcoin.ReaderWriterLock; @@ -33,11 +34,16 @@ public InMemoryCoinView(HashHeightPair tipHash) } /// - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { throw new NotImplementedException(); } + /// + public void Sync(ChainIndexer chainIndexer) + { + } + /// public HashHeightPair GetTipHash() { diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs index c03e4b0f00..bf4d5c26b1 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs @@ -25,6 +25,7 @@ public override Task RunAsync(RuleContext context) // unless the coinview threshold is reached. this.Logger.LogDebug("Saving coinview changes."); var utxoRuleContext = context as UtxoRuleContext; + this.PowParent.UtxoSet.Sync(this.Parent.ChainIndexer); this.PowParent.UtxoSet.SaveChanges(utxoRuleContext.UnspentOutputSet.GetCoins(), new HashHeightPair(oldBlock), new HashHeightPair(nextBlock)); return Task.CompletedTask; @@ -66,7 +67,7 @@ public override Task RunAsync(RuleContext context) { // Check that the current block has not been reorged. // Catching a reorg at this point will not require a rewind. - if (context.ValidationContext.BlockToValidate.Header.HashPrevBlock != this.Parent.ChainState.ConsensusTip.HashBlock) + if (context.ValidationContext.BlockToValidate.Header.HashPrevBlock != this.PowParent.UtxoSet.GetTipHash().Hash) { this.Logger.LogDebug("Reorganization detected."); ConsensusErrors.InvalidPrevTip.Throw(); diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs index 60dd9519b7..92319d73db 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs @@ -67,7 +67,7 @@ public override void Initialize(ChainedHeader chainTip) { base.Initialize(chainTip); - this.UtxoSet.Initialize(chainTip, this.ChainIndexer); + this.UtxoSet.Initialize(chainTip, this.ChainIndexer, this); } public override async Task FullValidationAsync(ChainedHeader header, Block block) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs index 286478d185..79e782b89e 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using NBitcoin; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Features.MemoryPool.Interfaces; @@ -49,11 +50,16 @@ public MempoolCoinView(Network network, ICoinView inner, ITxMempool memPool, Sch this.Set = new UnspentOutputSet(); } - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { throw new NotImplementedException(); } + /// + public void Sync(ChainIndexer chainIndexer) + { + } + /// /// Gets the unspent transaction output set. /// diff --git a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs index 35fd7898fa..4c8f8bcfa6 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using NBitcoin; +using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Utilities; using ReaderWriterLock = NBitcoin.ReaderWriterLock; @@ -34,11 +35,16 @@ public TestInMemoryCoinView(HashHeightPair tipHash) } /// - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer) + public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { throw new NotImplementedException(); } + /// + public void Sync(ChainIndexer chainIndexer) + { + } + /// public HashHeightPair GetTipHash() { From e6453157ef88519673eefef3049bf2f5959989a2 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 12 Jul 2022 16:50:42 +1000 Subject: [PATCH 21/44] Merge CoinDb implementations (#1017) * Merge CoinDb implementations * Reduce changes * Update db variable name * Update db variable name * Add comments to SeekToLast --- .../Coindb/{LeveldbCoindb.cs => Coindb.cs} | 106 +++-- .../CoinViews/Coindb/ICoindb.cs | 8 + .../CoinViews/Coindb/RocksDbCoindb.cs | 385 ------------------ .../FullNodeBuilderConsensusExtension.cs | 7 +- .../NodeContext.cs | 5 +- src/Stratis.Bitcoin/Database/LevelDb.cs | 12 +- src/Stratis.Bitcoin/Database/RocksDb.cs | 6 +- 7 files changed, 76 insertions(+), 453 deletions(-) rename src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/{LeveldbCoindb.cs => Coindb.cs} (72%) delete mode 100644 src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs similarity index 72% rename from src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs rename to src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index 2584bab096..be0cd8603a 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -2,19 +2,20 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Logging; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.CoinViews { /// - /// Persistent implementation of coinview using the dBreeze database engine. + /// Persistent implementation of coinview using an database engine. /// - public class LevelDbCoindb : ICoindb, IStakedb, IDisposable + /// A database supporting the interface. + public class Coindb : ICoindb, IStakedb, IDisposable where T : IDb, new() { /// Database key under which the block hash of the coin view's current tip is stored. private static readonly byte[] blockHashKey = new byte[0]; @@ -41,19 +42,19 @@ public class LevelDbCoindb : ICoindb, IStakedb, IDisposable private BackendPerformanceSnapshot latestPerformanceSnapShot; /// Access to dBreeze database. - private DB leveldb; + private IDb coinDb; private readonly DBreezeSerializer dBreezeSerializer; private const int MaxRewindBatchSize = 10000; - public LevelDbCoindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, + public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) : this(network, dataFolder.CoindbPath, dateTimeProvider, nodeStats, dBreezeSerializer) { } - public LevelDbCoindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, + public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) { Guard.NotNull(network, nameof(network)); @@ -72,38 +73,38 @@ public LevelDbCoindb(Network network, string dataFolder, IDateTimeProvider dateT public void Initialize(ChainedHeader chainTip) { // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, this.dataFolder); + this.coinDb = new T(); + this.coinDb.Open(this.dataFolder); // Check if key bytes are in the wrong endian order. HashHeightPair current = this.GetTipHash(); if (current != null) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height)).ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height)); // Fix the table if required. if (row != null) { // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height - 1)).ToArray()) : new byte[] { }; + byte[] row2 = (current.Height > 1) ? this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; if (row2 != null) { this.logger.LogInformation("Fixing the coin db."); var rows = new Dictionary(); - using (var iterator = this.leveldb.CreateIterator()) + using (var iterator = this.coinDb.GetIterator(rewindTable)) { - iterator.Seek(new byte[] { rewindTable }); + iterator.Seek(new byte[0]); while (iterator.IsValid()) { byte[] key = iterator.Key(); - if (key.Length != 5 || key[0] != rewindTable) + if (key.Length != 4) break; - int height = BitConverter.ToInt32(key, 1); + int height = BitConverter.ToInt32(key); rows[height] = iterator.Value(); @@ -111,19 +112,19 @@ public void Initialize(ChainedHeader chainTip) } } - using (var batch = new WriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { foreach (int height in rows.Keys.OrderBy(k => k)) { - batch.Delete(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height)).ToArray()); + batch.Delete(rewindTable, BitConverter.GetBytes(height)); } foreach (int height in rows.Keys.OrderBy(k => k)) { - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray(), rows[height]); + batch.Put(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray(), rows[height]); } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } } @@ -135,10 +136,10 @@ public void Initialize(ChainedHeader chainTip) if (this.GetTipHash() == null) { - using (var batch = new WriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -156,7 +157,7 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) { heightToCheck += 1; - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(heightToCheck).Reverse()).ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); if (row == null) break; @@ -173,17 +174,17 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) this.logger.LogInformation("Coin database integrity good."); } - private void SetBlockHash(WriteBatch batch, HashHeightPair nextBlockHash) + private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash) { this.persistedCoinviewTip = nextBlockHash; - batch.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes()); + batch.Put(blockTable, blockHashKey, nextBlockHash.ToBytes()); } public HashHeightPair GetTipHash() { if (this.persistedCoinviewTip == null) { - var row = this.leveldb.Get(new byte[] { blockTable }.Concat(blockHashKey).ToArray()); + var row = this.coinDb.Get(blockTable, blockHashKey); if (row != null) { this.persistedCoinviewTip = new HashHeightPair(); @@ -204,7 +205,7 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) foreach (OutPoint outPoint in utxos) { - byte[] row = this.leveldb.Get(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); + byte[] row = this.coinDb.Get(coinsTable, outPoint.ToBytes()); Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); @@ -220,7 +221,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB { int insertedEntities = 0; - using (var batch = new WriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -239,7 +240,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB if (coin.Coins == null) { this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint); - batch.Delete(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray()); + batch.Delete(coinsTable, coin.OutPoint.ToBytes()); } else { @@ -254,7 +255,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB var coin = toInsert[i]; this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count); - batch.Put(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(coin.Coins)); + batch.Put(coinsTable, coin.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(coin.Coins)); } if (rewindDataList != null) @@ -265,13 +266,13 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(nextRewindIndex).Reverse()).ToArray(), this.dBreezeSerializer.Serialize(rewindData)); + batch.Put(rewindTable, BitConverter.GetBytes(nextRewindIndex).Reverse().ToArray(), this.dBreezeSerializer.Serialize(rewindData)); } } insertedEntities += unspentOutputs.Count; this.SetBlockHash(batch, nextBlockHash); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -282,22 +283,21 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB public int GetMinRewindHeight() { // Find the first row with a rewind table key prefix. - using (var iterator = this.leveldb.CreateIterator()) + using (var iterator = this.coinDb.GetIterator(rewindTable)) { - iterator.Seek(new byte[] { rewindTable }); + iterator.Seek(new byte[0]); if (!iterator.IsValid()) return -1; byte[] key = iterator.Key(); - if (key.Length != 5 || key[0] != rewindTable) + if (key.Length != 4) return -1; - return BitConverter.ToInt32(key.SafeSubarray(1, 4).Reverse().ToArray()); + return BitConverter.ToInt32(key.SafeSubarray(0, 4).Reverse().ToArray()); } } - /// public HashHeightPair Rewind(HashHeightPair target) { HashHeightPair current = this.GetTipHash(); @@ -308,36 +308,36 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = new WriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); - batch.Delete(BitConverter.GetBytes(height)); + batch.Delete(rewindTable, BitConverter.GetBytes(height)); var rewindData = this.dBreezeSerializer.Deserialize(row); foreach (OutPoint outPoint in rewindData.OutputsToRemove) { this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); + batch.Delete(coinsTable, outPoint.ToBytes()); } foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) { this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); - batch.Put(new byte[] { coinsTable }.Concat(rewindDataOutput.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); + batch.Put(coinsTable, rewindDataOutput.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); } res = rewindData.PreviousBlockHash; } this.SetBlockHash(batch, res); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } return res; @@ -345,41 +345,33 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) public RewindData GetRewindData(int height) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); return row != null ? this.dBreezeSerializer.Deserialize(row) : null; } - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. public void PutStake(IEnumerable stakeEntries) { - using (var batch = new WriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { foreach (StakeItem stakeEntry in stakeEntries) { if (!stakeEntry.InStore) { - batch.Put(new byte[] { stakeTable }.Concat(stakeEntry.BlockId.ToBytes(false)).ToArray(), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); + batch.Put(stakeTable, stakeEntry.BlockId.ToBytes(false), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); stakeEntry.InStore = true; } } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } - /// - /// Retrieves POS blocks information from the database. - /// - /// List of partially initialized POS block information that is to be fully initialized with the values from the database. public void GetStake(IEnumerable blocklist) { foreach (StakeItem blockStake in blocklist) { this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.leveldb.Get(new byte[] { stakeTable }.Concat(blockStake.BlockId.ToBytes(false)).ToArray()); + byte[] stakeRow = this.coinDb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); if (stakeRow != null) { @@ -391,7 +383,7 @@ public void GetStake(IEnumerable blocklist) private void AddBenchStats(StringBuilder log) { - log.AppendLine(">> Leveldb Bench"); + log.AppendLine(">> Coindb Bench"); BackendPerformanceSnapshot snapShot = this.performanceCounter.Snapshot(); @@ -406,7 +398,7 @@ private void AddBenchStats(StringBuilder log) /// public void Dispose() { - this.leveldb.Dispose(); + this.coinDb.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index 32b23b5f49..f8c664d6b6 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -80,8 +80,16 @@ public interface ICoindb public interface IStakedb : ICoindb { + /// + /// Persists unsaved POS blocks information to the database. + /// + /// List of POS block information to be examined and persists if unsaved. void PutStake(IEnumerable stakeEntries); + /// + /// Retrieves POS blocks information from the database. + /// + /// List of partially initialized POS block information that is to be fully initialized with the values from the database. void GetStake(IEnumerable blocklist); } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs deleted file mode 100644 index 6e55bcd059..0000000000 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs +++ /dev/null @@ -1,385 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Microsoft.Extensions.Logging; -using NBitcoin; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.Consensus.CoinViews -{ - /// - /// Persistent implementation of coinview using dBreeze database. - /// - public class RocksDbCoindb : ICoindb, IStakedb, IDisposable - { - /// Database key under which the block hash of the coin view's current tip is stored. - private static readonly byte[] blockHashKey = new byte[0]; - - private static readonly byte coinsTable = 1; - private static readonly byte blockTable = 2; - private static readonly byte rewindTable = 3; - private static readonly byte stakeTable = 4; - - private readonly string dataFolder; - - /// Hash of the block which is currently the tip of the coinview. - private HashHeightPair persistedCoinviewTip; - private readonly DBreezeSerializer dBreezeSerializer; - private DbOptions dbOptions; - private RocksDb rocksDb; - private BackendPerformanceSnapshot latestPerformanceSnapShot; - private readonly ILogger logger; - private readonly Network network; - private readonly BackendPerformanceCounter performanceCounter; - - public RocksDbCoindb( - Network network, - DataFolder dataFolder, - IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, - DBreezeSerializer dBreezeSerializer) - { - this.dataFolder = dataFolder.CoindbPath; - this.dBreezeSerializer = dBreezeSerializer; - this.logger = LogManager.GetCurrentClassLogger(); - this.network = network; - this.performanceCounter = new BackendPerformanceCounter(dateTimeProvider); - - if (nodeStats.DisplayBenchStats) - nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); - } - - public void Initialize(ChainedHeader chainTip) - { - this.dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder); - - // Check if key bytes are in the wrong endian order. - HashHeightPair current = this.GetTipHash(); - if (current != null) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height)).ToArray()); - - // Fix the table if required. - if (row != null) - { - // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height - 1)).ToArray()) : new byte[] { }; - if (row2 != null) - { - this.logger.LogInformation("Fixing the coin db."); - - var rows = new Dictionary(); - - using (var iterator = this.rocksDb.NewIterator()) - { - iterator.Seek(new byte[] { rewindTable }); - - while (iterator.Valid()) - { - byte[] key = iterator.Key(); - - if (key.Length != 5 || key[0] != rewindTable) - break; - - int height = BitConverter.ToInt32(key, 1); - - rows[height] = iterator.Value(); - - iterator.Next(); - } - } - - using (var batch = new WriteBatch()) - { - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Delete(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height)).ToArray()); - } - - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray(), rows[height]); - } - - this.rocksDb.Write(batch); - } - } - } - } - - EnsureCoinDatabaseIntegrity(chainTip); - - Block genesis = this.network.GetGenesis(); - - if (this.GetTipHash() == null) - this.SetBlockHash(new HashHeightPair(genesis.GetHash(), 0)); - - this.logger.LogInformation("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip); - } - - private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) - { - this.logger.LogInformation("Checking coin database integrity..."); - - var heightToCheck = chainTip.Height; - - // Find the height up to where rewind data is stored above chain tip. - do - { - heightToCheck += 1; - - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(heightToCheck).Reverse()).ToArray()); - if (row == null) - break; - - } while (true); - - using (var batch = new WriteBatch()) - { - for (int height = heightToCheck - 1; height > chainTip.Height; height--) - { - this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{chainTip}'."); - RewindInternal(batch, height); - } - } - - this.logger.LogInformation("Coin database integrity good."); - } - - private void SetBlockHash(HashHeightPair nextBlockHash) - { - this.persistedCoinviewTip = nextBlockHash; - this.rocksDb.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes()); - } - - public HashHeightPair GetTipHash() - { - if (this.persistedCoinviewTip == null) - { - var row = this.rocksDb.Get(new byte[] { blockTable }.Concat(blockHashKey).ToArray()); - if (row != null) - { - this.persistedCoinviewTip = new HashHeightPair(); - this.persistedCoinviewTip.FromBytes(row); - } - } - - return this.persistedCoinviewTip; - } - - public FetchCoinsResponse FetchCoins(OutPoint[] utxos) - { - FetchCoinsResponse res = new FetchCoinsResponse(); - - using (new StopwatchDisposable(o => this.performanceCounter.AddQueryTime(o))) - { - this.performanceCounter.AddQueriedEntities(utxos.Length); - - foreach (OutPoint outPoint in utxos) - { - byte[] row = this.rocksDb.Get(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); - Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; - - this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); - - res.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, outputs)); - } - } - - return res; - } - - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) - { - int insertedEntities = 0; - - using (var batch = new WriteBatch()) - { - using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) - { - HashHeightPair current = this.GetTipHash(); - if (current != oldBlockHash) - { - this.logger.LogError("(-)[BLOCKHASH_MISMATCH]"); - throw new InvalidOperationException("Invalid oldBlockHash"); - } - - // Here we'll add items to be inserted in a second pass. - List toInsert = new List(); - - foreach (var coin in unspentOutputs.OrderBy(utxo => utxo.OutPoint, new OutPointComparer())) - { - if (coin.Coins == null) - { - this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint); - batch.Delete(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray()); - } - else - { - // Add the item to another list that will be used in the second pass. - // This is for performance reasons: dBreeze is optimized to run the same kind of operations, sorted. - toInsert.Add(coin); - } - } - - for (int i = 0; i < toInsert.Count; i++) - { - var coin = toInsert[i]; - this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count); - - batch.Put(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(coin.Coins)); - } - - if (rewindDataList != null) - { - foreach (RewindData rewindData in rewindDataList) - { - var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1; - - this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(nextRewindIndex).Reverse()).ToArray(), this.dBreezeSerializer.Serialize(rewindData)); - } - } - - insertedEntities += unspentOutputs.Count; - this.rocksDb.Write(batch); - - this.SetBlockHash(nextBlockHash); - } - } - - this.performanceCounter.AddInsertedEntities(insertedEntities); - } - - - /// - public int GetMinRewindHeight() - { - // Find the first row with a rewind table key prefix. - using (var iterator = this.rocksDb.NewIterator()) - { - iterator.Seek(new byte[] { rewindTable }); - if (!iterator.Valid()) - return -1; - - byte[] key = iterator.Key(); - - if (key.Length != 5 || key[0] != rewindTable) - return -1; - - return BitConverter.ToInt32(key.SafeSubarray(1, 4).Reverse().ToArray()); - } - } - - /// - public HashHeightPair Rewind(HashHeightPair target) - { - using (var batch = new WriteBatch()) - { - HashHeightPair current = this.GetTipHash(); - return RewindInternal(batch, current.Height); - } - } - - private HashHeightPair RewindInternal(WriteBatch batch, int height) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); - - if (row == null) - throw new InvalidOperationException($"No rewind data found for block at height {height}."); - - batch.Delete(BitConverter.GetBytes(height)); - - var rewindData = this.dBreezeSerializer.Deserialize(row); - - foreach (OutPoint outPoint in rewindData.OutputsToRemove) - { - this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); - } - - foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) - { - this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); - batch.Put(new byte[] { coinsTable }.Concat(rewindDataOutput.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); - } - - this.rocksDb.Write(batch); - - this.SetBlockHash(rewindData.PreviousBlockHash); - - return rewindData.PreviousBlockHash; - } - - public RewindData GetRewindData(int height) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); - return row != null ? this.dBreezeSerializer.Deserialize(row) : null; - } - - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. - public void PutStake(IEnumerable stakeEntries) - { - using var batch = new WriteBatch(); - { - foreach (StakeItem stakeEntry in stakeEntries) - { - if (!stakeEntry.InStore) - { - batch.Put(new byte[] { stakeTable }.Concat(stakeEntry.BlockId.ToBytes(false)).ToArray(), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); - stakeEntry.InStore = true; - } - } - - this.rocksDb.Write(batch); - } - } - - /// - /// Retrieves POS blocks information from the database. - /// - /// List of partially initialized POS block information that is to be fully initialized with the values from the database. - public void GetStake(IEnumerable blocklist) - { - foreach (StakeItem blockStake in blocklist) - { - this.logger.LogDebug("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.rocksDb.Get(new byte[] { stakeTable }.Concat(blockStake.BlockId.ToBytes(false)).ToArray()); - - if (stakeRow != null) - { - blockStake.BlockStake = this.dBreezeSerializer.Deserialize(stakeRow); - blockStake.InStore = true; - } - } - } - - private void AddBenchStats(StringBuilder log) - { - log.AppendLine("======RocksDb Bench======"); - - BackendPerformanceSnapshot snapShot = this.performanceCounter.Snapshot(); - - if (this.latestPerformanceSnapShot == null) - log.AppendLine(snapShot.ToString()); - else - log.AppendLine((snapShot - this.latestPerformanceSnapShot).ToString()); - - this.latestPerformanceSnapShot = snapShot; - } - - public void Dispose() - { - this.rocksDb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index fb3f89f155..06a751fb35 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -4,6 +4,7 @@ using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Features.Consensus.Interfaces; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; @@ -87,15 +88,15 @@ public static void ConfigureCoinDatabaseImplementation(this IServiceCollection s break; case DbType.Leveldb: - services.AddSingleton(); + services.AddSingleton>(); break; case DbType.RocksDb: - services.AddSingleton(); + services.AddSingleton>(); break; default: - services.AddSingleton(); + services.AddSingleton>(); break; } } diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index 5efc05f99a..fb005f0b45 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -5,6 +5,7 @@ using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Tests.Common; @@ -27,7 +28,7 @@ public NodeContext(object caller, string name, Network network) this.FolderName = TestBase.CreateTestDir(caller, name); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new LevelDbCoindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); + this.Coindb = new Coindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); this.Coindb.Initialize(new ChainedHeader(network.GetGenesis().Header, network.GenesisHash, 0)); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -65,7 +66,7 @@ public void ReloadPersistentCoinView(ChainedHeader chainTip) this.cleanList.Remove((IDisposable)this.Coindb); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new LevelDbCoindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); + this.Coindb = new Coindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); this.Coindb.Initialize(chainTip); this.cleanList.Add((IDisposable)this.Coindb); diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs index 67f82ac593..1fa5b8796e 100644 --- a/src/Stratis.Bitcoin/Database/LevelDb.cs +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -1,5 +1,6 @@ using System.Linq; using LevelDB; +using NBitcoin; namespace Stratis.Bitcoin.Database { @@ -88,8 +89,8 @@ public void SeekToLast() { if (this.table != 255) { - // First seek past the last record in the table by attempting to seek to the start of the next table (if any). - this.iterator.Seek(new[] { (byte)(this.table + 1) }); + // First seek past the last record in the table by attempting to seek to the start of the next table (if any). + this.iterator.Seek(new[] { (byte)(this.table + 1) }); // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. if (this.iterator.IsValid()) @@ -99,8 +100,11 @@ public void SeekToLast() } } - // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. - this.iterator.SeekToLast(); + // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. + this.iterator.SeekToLast(); + else + // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. + this.iterator.Prev(); } public void Next() diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index 39092245f8..59b737bda0 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -90,7 +90,7 @@ public void SeekToLast() if (this.table != 255) { // First seek past the last record in the table by attempting to seek to the start of the next table (if any). - this.iterator.Seek(new[] { (byte)(this.table + 1) }); + this.iterator.Seek(new[] { (byte)(this.table + 1) }); // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. if (this.iterator.Valid()) @@ -101,7 +101,9 @@ public void SeekToLast() } // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. - this.iterator.SeekToLast(); + this.iterator.SeekToLast(); + else + this.iterator.Prev(); } public void Next() From 1a3c4fa42ad4f592ed7960d8442259c9593601e3 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 25 Jul 2022 14:48:15 +1000 Subject: [PATCH 22/44] Combine ChainStore implementations (#1025) * Combine ChainStore implementations * Update tests --- .../CoinViewTests.cs | 3 +- .../Base/ChainRepositoryTest.cs | 5 +- src/Stratis.Bitcoin/Base/BaseFeature.cs | 5 +- src/Stratis.Bitcoin/Database/LevelDb.cs | 12 +- src/Stratis.Bitcoin/Database/RocksDb.cs | 6 +- .../{LevelDbChainStore.cs => ChainStore.cs} | 28 ++-- .../ChainStores/RocksDbChainStore.cs | 134 ------------------ 7 files changed, 30 insertions(+), 163 deletions(-) rename src/Stratis.Bitcoin/Persistence/ChainStores/{LevelDbChainStore.cs => ChainStore.cs} (80%) delete mode 100644 src/Stratis.Bitcoin/Persistence/ChainStores/RocksDbChainStore.cs diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index dc3ae155a1..eed6363a16 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -10,6 +10,7 @@ using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.IntegrationTests.Common; @@ -282,7 +283,7 @@ public void CanSaveChainIncrementally() var chain = new ChainIndexer(this.regTest); var data = new DataFolder(TestBase.CreateTestDir(this)); - var chainStore = new LevelDbChainStore(this.network, data, chain); + var chainStore = new ChainStore(this.network, data, chain); chain[0].SetChainStore(chainStore); using (var repo = new ChainRepository(chainStore)) diff --git a/src/Stratis.Bitcoin.Tests/Base/ChainRepositoryTest.cs b/src/Stratis.Bitcoin.Tests/Base/ChainRepositoryTest.cs index a6a406cb78..31d53eafa8 100644 --- a/src/Stratis.Bitcoin.Tests/Base/ChainRepositoryTest.cs +++ b/src/Stratis.Bitcoin.Tests/Base/ChainRepositoryTest.cs @@ -5,6 +5,7 @@ using NBitcoin; using Stratis.Bitcoin.Base; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Persistence.ChainStores; using Stratis.Bitcoin.Tests.Common; @@ -25,7 +26,7 @@ public void SaveChainToDisk() var chain = new ChainIndexer(KnownNetworks.StraxRegTest); this.AppendBlock(chain); - using (var repo = new ChainRepository(new LevelDbChainStore(chain.Network, new DataFolder(dir), chain))) + using (var repo = new ChainRepository(new ChainStore(chain.Network, new DataFolder(dir), chain))) { repo.SaveAsync(chain).GetAwaiter().GetResult(); } @@ -85,7 +86,7 @@ public void LoadChainFromDisk() } } - var chainStore = new LevelDbChainStore(chain.Network, new DataFolder(dir), chain); + var chainStore = new ChainStore(chain.Network, new DataFolder(dir), chain); using (var repo = new ChainRepository(chainStore)) { var testChain = new ChainIndexer(KnownNetworks.StraxRegTest); diff --git a/src/Stratis.Bitcoin/Base/BaseFeature.cs b/src/Stratis.Bitcoin/Base/BaseFeature.cs index 31f50d4ab6..5bd9863fb6 100644 --- a/src/Stratis.Bitcoin/Base/BaseFeature.cs +++ b/src/Stratis.Bitcoin/Base/BaseFeature.cs @@ -18,6 +18,7 @@ using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Consensus.Rules; using Stratis.Bitcoin.Consensus.Validators; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.EventBus; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.P2P; @@ -459,13 +460,13 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil if (dbType == DbType.Leveldb) { - chainStore = new LevelDbChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); + chainStore = new ChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); services.AddSingleton(); } if (dbType == DbType.RocksDb) { - chainStore = new RocksDbChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); + chainStore = new ChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); services.AddSingleton(); } diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs index 1fa5b8796e..67f82ac593 100644 --- a/src/Stratis.Bitcoin/Database/LevelDb.cs +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -1,6 +1,5 @@ using System.Linq; using LevelDB; -using NBitcoin; namespace Stratis.Bitcoin.Database { @@ -89,8 +88,8 @@ public void SeekToLast() { if (this.table != 255) { - // First seek past the last record in the table by attempting to seek to the start of the next table (if any). - this.iterator.Seek(new[] { (byte)(this.table + 1) }); + // First seek past the last record in the table by attempting to seek to the start of the next table (if any). + this.iterator.Seek(new[] { (byte)(this.table + 1) }); // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. if (this.iterator.IsValid()) @@ -100,11 +99,8 @@ public void SeekToLast() } } - // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. - this.iterator.SeekToLast(); - else - // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. - this.iterator.Prev(); + // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. + this.iterator.SeekToLast(); } public void Next() diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index 59b737bda0..39092245f8 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -90,7 +90,7 @@ public void SeekToLast() if (this.table != 255) { // First seek past the last record in the table by attempting to seek to the start of the next table (if any). - this.iterator.Seek(new[] { (byte)(this.table + 1) }); + this.iterator.Seek(new[] { (byte)(this.table + 1) }); // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. if (this.iterator.Valid()) @@ -101,9 +101,7 @@ public void SeekToLast() } // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. - this.iterator.SeekToLast(); - else - this.iterator.Prev(); + this.iterator.SeekToLast(); } public void Next() diff --git a/src/Stratis.Bitcoin/Persistence/ChainStores/LevelDbChainStore.cs b/src/Stratis.Bitcoin/Persistence/ChainStores/ChainStore.cs similarity index 80% rename from src/Stratis.Bitcoin/Persistence/ChainStores/LevelDbChainStore.cs rename to src/Stratis.Bitcoin/Persistence/ChainStores/ChainStore.cs index e038293326..3422db9a43 100644 --- a/src/Stratis.Bitcoin/Persistence/ChainStores/LevelDbChainStore.cs +++ b/src/Stratis.Bitcoin/Persistence/ChainStores/ChainStore.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; -using LevelDB; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Persistence.ChainStores { - public class LevelDbChainStore : IChainStore + public class ChainStore : IChainStore where T : IDb, new() { private readonly Network network; @@ -19,11 +19,11 @@ public class LevelDbChainStore : IChainStore /// private readonly MemoryCountCache headers; - private readonly DB leveldb; + private readonly IDb db; private readonly object locker; - public LevelDbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer) + public ChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer) { this.network = network; this.ChainIndexer = chainIndexer; @@ -31,8 +31,8 @@ public LevelDbChainStore(Network network, DataFolder dataFolder, ChainIndexer ch this.locker = new object(); // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, dataFolder.ChainPath); + this.db = new T(); + this.db.Open(dataFolder.ChainPath); } public ChainIndexer ChainIndexer { get; } @@ -49,7 +49,7 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) lock (this.locker) { - bytes = this.leveldb.Get(HeaderTableName, bytes); + bytes = this.db.Get(HeaderTableName, bytes); } if (bytes == null) @@ -82,7 +82,11 @@ public bool PutHeader(BlockHeader blockHeader) lock (this.locker) { - this.leveldb.Put(HeaderTableName, blockHeader.GetHash().ToBytes(), blockHeader.ToBytes(consensusFactory)); + using (var batch = this.db.GetWriteBatch()) + { + batch.Put(HeaderTableName, blockHeader.GetHash().ToBytes(), blockHeader.ToBytes(consensusFactory)); + batch.Write(); + } } return true; @@ -94,7 +98,7 @@ public ChainData GetChainData(int height) lock (this.locker) { - bytes = this.leveldb.Get(ChainTableName, BitConverter.GetBytes(height)); + bytes = this.db.Get(ChainTableName, BitConverter.GetBytes(height)); } if (bytes == null) @@ -110,7 +114,7 @@ public ChainData GetChainData(int height) public void PutChainData(IEnumerable items) { - using (var batch = new WriteBatch()) + using (var batch = this.db.GetWriteBatch()) { foreach (var item in items) { @@ -119,14 +123,14 @@ public void PutChainData(IEnumerable items) lock (this.locker) { - this.leveldb.Write(batch, new WriteOptions { Sync = true }); + batch.Write(); } } } public void Dispose() { - this.leveldb?.Dispose(); + this.db?.Dispose(); } } } diff --git a/src/Stratis.Bitcoin/Persistence/ChainStores/RocksDbChainStore.cs b/src/Stratis.Bitcoin/Persistence/ChainStores/RocksDbChainStore.cs deleted file mode 100644 index 44f6da3c23..0000000000 --- a/src/Stratis.Bitcoin/Persistence/ChainStores/RocksDbChainStore.cs +++ /dev/null @@ -1,134 +0,0 @@ -using System; -using System.Collections.Generic; -using NBitcoin; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Persistence.ChainStores -{ - /// - /// Rocksdb implementation of the chain storage - /// - public sealed class RocksDbChainStore : IChainStore - { - internal static readonly byte ChainTableName = 1; - internal static readonly byte HeaderTableName = 2; - - private readonly string dataFolder; - private readonly Network network; - - /// Headers that are close to the tip. - private readonly MemoryCountCache headers; - - private readonly object locker; - private readonly DbOptions dbOptions; - private readonly RocksDb rocksDb; - - public RocksDbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer) - { - this.dataFolder = dataFolder.ChainPath; - this.network = network; - this.ChainIndexer = chainIndexer; - this.headers = new MemoryCountCache(601); - this.locker = new object(); - - this.dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder); - } - - public ChainIndexer ChainIndexer { get; } - - public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) - { - if (this.headers.TryGetValue(hash, out BlockHeader blockHeader)) - { - return blockHeader; - } - - // TODO: Bring in uint256 span optimisations - byte[] bytes = hash.ToBytes(); - - lock (this.locker) - { - bytes = this.rocksDb.Get(HeaderTableName, bytes); - } - - if (bytes == null) - { - throw new ApplicationException("Header must exist if requested"); - } - - blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); - blockHeader.FromBytes(bytes, this.network.Consensus.ConsensusFactory); - - // If the header is 500 blocks behind tip or 100 blocks ahead cache it. - if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 100)) - this.headers.AddOrUpdate(hash, blockHeader); - - return blockHeader; - } - - public bool PutHeader(BlockHeader blockHeader) - { - ConsensusFactory consensusFactory = this.network.Consensus.ConsensusFactory; - - if (blockHeader is ProvenBlockHeader) - { - // If ProvenBlockHeader copy the header parameters. - BlockHeader newHeader = consensusFactory.CreateBlockHeader(); - newHeader.CopyFields(blockHeader); - - blockHeader = newHeader; - } - - lock (this.locker) - { - this.rocksDb.Put(HeaderTableName, blockHeader.GetHash().ToBytes(), blockHeader.ToBytes(consensusFactory)); - } - - return true; - } - - public ChainData GetChainData(int height) - { - byte[] bytes = null; - - lock (this.locker) - { - bytes = this.rocksDb.Get(ChainTableName, BitConverter.GetBytes(height)); - } - - if (bytes == null) - { - return null; - } - - var data = new ChainData(); - data.FromBytes(bytes, this.network.Consensus.ConsensusFactory); - - return data; - } - - public void PutChainData(IEnumerable items) - { - using (var batch = new WriteBatch()) - { - foreach (var item in items) - { - batch.Put(ChainTableName, BitConverter.GetBytes(item.Height), item.Data.ToBytes(this.network.Consensus.ConsensusFactory)); - } - - lock (this.locker) - { - this.rocksDb.Write(batch); - } - } - } - - public void Dispose() - { - this.rocksDb?.Dispose(); - } - } -} \ No newline at end of file From 0eb359e1d06db269f9610af95fcc55e5fab59c9e Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 4 Aug 2022 16:02:50 +1000 Subject: [PATCH 23/44] Combine KeyValueStore implementations (#1029) * Combine KeyValueStore implementations * Update tests (cherry picked from commit 857eb537de1de6d04f8e4aa1bb9f8582fb93f9d2) --- .../Base/TipsManagerTests.cs | 5 +- .../FinalizedBlockInfoRepositoryTest.cs | 5 +- src/Stratis.Bitcoin/Base/BaseFeature.cs | 4 +- src/Stratis.Bitcoin/Database/IDb.cs | 28 +++++ src/Stratis.Bitcoin/Database/LevelDb.cs | 50 +++++++- src/Stratis.Bitcoin/Database/RocksDb.cs | 45 ++++++- ...lueRepository.cs => KeyValueRepository.cs} | 66 +++++----- .../RocksDbKeyValueRepository.cs | 113 ------------------ .../Distribution/RewardClaimerTests.cs | 5 +- .../Conversion/ReplenishmentKeyValueStore.cs | 3 +- .../ConversionRequestFeeKeyValueStore.cs | 3 +- 11 files changed, 164 insertions(+), 163 deletions(-) rename src/Stratis.Bitcoin/Persistence/KeyValueStores/{LevelDbKeyValueRepository.cs => KeyValueRepository.cs} (62%) delete mode 100644 src/Stratis.Bitcoin/Persistence/KeyValueStores/RocksDbKeyValueRepository.cs diff --git a/src/Stratis.Bitcoin.Tests/Base/TipsManagerTests.cs b/src/Stratis.Bitcoin.Tests/Base/TipsManagerTests.cs index fc7573af79..616f3686b5 100644 --- a/src/Stratis.Bitcoin.Tests/Base/TipsManagerTests.cs +++ b/src/Stratis.Bitcoin.Tests/Base/TipsManagerTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Base; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Persistence.KeyValueStores; using Stratis.Bitcoin.Tests.Common; using Stratis.Bitcoin.Utilities; @@ -14,7 +15,7 @@ namespace Stratis.Bitcoin.Tests.Base public class TipsManagerTests : TestBase { private readonly LoggerFactory loggerFactory; - private readonly LevelDbKeyValueRepository keyValueRepo; + private readonly KeyValueRepository keyValueRepo; private readonly ITipsManager tipsManager; private readonly List mainChainHeaders; @@ -23,7 +24,7 @@ public TipsManagerTests() : base(KnownNetworks.StraxMain) { this.loggerFactory = new LoggerFactory(); string dir = CreateTestDir(this); - this.keyValueRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); + this.keyValueRepo = new KeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); this.tipsManager = new TipsManager(this.keyValueRepo, this.loggerFactory); diff --git a/src/Stratis.Bitcoin.Tests/Consensus/FinalizedBlockInfoRepositoryTest.cs b/src/Stratis.Bitcoin.Tests/Consensus/FinalizedBlockInfoRepositoryTest.cs index 054f7afc3a..cb1494432f 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/FinalizedBlockInfoRepositoryTest.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/FinalizedBlockInfoRepositoryTest.cs @@ -4,6 +4,7 @@ using NBitcoin; using Stratis.Bitcoin.AsyncWork; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Persistence.KeyValueStores; using Stratis.Bitcoin.Tests.Common; using Stratis.Bitcoin.Utilities; @@ -24,7 +25,7 @@ public FinalizedBlockInfoRepositoryTest() : base(KnownNetworks.StraxRegTest) public async Task FinalizedHeightSavedOnDiskAsync() { string dir = CreateTestDir(this); - var kvRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); + var kvRepo = new KeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); var asyncMock = new Mock(); asyncMock.Setup(a => a.RegisterTask(It.IsAny(), It.IsAny())); @@ -45,7 +46,7 @@ public async Task FinalizedHeightSavedOnDiskAsync() public async Task FinalizedHeightCantBeDecreasedAsync() { string dir = CreateTestDir(this); - var kvRepo = new LevelDbKeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); + var kvRepo = new KeyValueRepository(dir, new DBreezeSerializer(this.Network.Consensus.ConsensusFactory)); var asyncMock = new Mock(); asyncMock.Setup(a => a.RegisterTask(It.IsAny(), It.IsAny())); diff --git a/src/Stratis.Bitcoin/Base/BaseFeature.cs b/src/Stratis.Bitcoin/Base/BaseFeature.cs index 5bd9863fb6..bf3fb3eed7 100644 --- a/src/Stratis.Bitcoin/Base/BaseFeature.cs +++ b/src/Stratis.Bitcoin/Base/BaseFeature.cs @@ -461,13 +461,13 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil if (dbType == DbType.Leveldb) { chainStore = new ChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); - services.AddSingleton(); + services.AddSingleton>(); } if (dbType == DbType.RocksDb) { chainStore = new ChainStore(fullNodeBuilder.Network, fullNodeBuilder.NodeSettings.DataFolder, chainIndexer); - services.AddSingleton(); + services.AddSingleton>(); } chainIndexer[0].SetChainStore(chainStore); diff --git a/src/Stratis.Bitcoin/Database/IDb.cs b/src/Stratis.Bitcoin/Database/IDb.cs index 807271a29e..4dea6b6023 100644 --- a/src/Stratis.Bitcoin/Database/IDb.cs +++ b/src/Stratis.Bitcoin/Database/IDb.cs @@ -25,6 +25,13 @@ public interface IDb : IDisposable /// The value for the specified table and key. byte[] Get(byte table, byte[] key); + /// + /// Gets the value associated with a key. + /// + /// The key of the value to retrieve. + /// The value for the specified key. + byte[] Get(byte[] key); + /// /// Gets an iterator that allows iteration over keys in a table. /// @@ -32,6 +39,12 @@ public interface IDb : IDisposable /// See . IDbIterator GetIterator(byte table); + /// + /// Gets an iterator that allows iteration over keys. + /// + /// See . + IDbIterator GetIterator(); + /// /// Gets a batch that can be used to record changes that can be applied atomically. /// @@ -69,6 +82,21 @@ public interface IDbBatch : IDisposable /// This class for fluent operations. IDbBatch Delete(byte table, byte[] key); + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This class for fluent operations. + IDbBatch Put(byte[] key, byte[] value); + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table key that will be removed. + /// This class for fluent operations. + IDbBatch Delete(byte[] key); + /// /// Writes the recorded changes to the database. /// diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs index 67f82ac593..408ac2ebce 100644 --- a/src/Stratis.Bitcoin/Database/LevelDb.cs +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -15,6 +15,11 @@ public IDbIterator GetIterator(byte table) return new LevelDbIterator(table, this.db.CreateIterator()); } + public IDbIterator GetIterator() + { + return new LevelDbIterator(this.db.CreateIterator()); + } + public void Open(string dbPath) { this.dbPath = dbPath; @@ -35,6 +40,11 @@ public byte[] Get(byte table, byte[] key) return this.db.Get(new[] { table }.Concat(key).ToArray()); } + public byte[] Get(byte[] key) + { + return this.db.Get(key); + } + public void Dispose() { this.db.Dispose(); @@ -51,14 +61,30 @@ public LevelDbBatch(DB db) this.db = db; } + // Methods when using tables. + public IDbBatch Put(byte table, byte[] key, byte[] value) { - return (IDbBatch)this.Put(new[] { table }.Concat(key).ToArray(), value); + return this.Put(new[] { table }.Concat(key).ToArray(), value); } public IDbBatch Delete(byte table, byte[] key) { - return (IDbBatch)this.Delete(new[] { table }.Concat(key).ToArray()); + return this.Delete(new[] { table }.Concat(key).ToArray()); + } + + // Table-less operations. + + public new IDbBatch Put(byte[] key, byte[] value) + { + base.Put(key, value); + return this; + } + + public new IDbBatch Delete(byte[] key) + { + base.Delete(key); + return this; } public void Write() @@ -70,7 +96,7 @@ public void Write() /// A minimal LevelDb wrapper that makes it compliant with the interface. public class LevelDbIterator : IDbIterator { - private byte table; + private byte? table; private Iterator iterator; public LevelDbIterator(byte table, Iterator iterator) @@ -79,13 +105,25 @@ public LevelDbIterator(byte table, Iterator iterator) this.iterator = iterator; } + // Table-less constructor. + public LevelDbIterator(Iterator iterator) + { + this.iterator = iterator; + } + public void Seek(byte[] key) { - this.iterator.Seek(new[] { this.table }.Concat(key).ToArray()); + this.iterator.Seek(this.table.HasValue ? (new[] { this.table.Value }.Concat(key).ToArray()) : key); } public void SeekToLast() { + if (!this.table.HasValue) + { + this.iterator.SeekToLast(); + return; + } + if (this.table != 255) { // First seek past the last record in the table by attempting to seek to the start of the next table (if any). @@ -115,12 +153,12 @@ public void Prev() public bool IsValid() { - return this.iterator.IsValid() && this.iterator.Key()[0] == this.table; + return this.iterator.IsValid() && (!this.table.HasValue || this.iterator.Key()[0] == this.table); } public byte[] Key() { - return this.iterator.Key().Skip(1).ToArray(); + return this.table.HasValue ? this.iterator.Key().Skip(1).ToArray() : this.iterator.Key(); } public byte[] Value() diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index 39092245f8..74af155ef7 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -11,6 +11,11 @@ public class RocksDb : IDb private RocksDbSharp.RocksDb db; + public IDbIterator GetIterator() + { + return new RocksDbIterator(this.db.NewIterator()); + } + public IDbIterator GetIterator(byte table) { return new RocksDbIterator(table, this.db.NewIterator()); @@ -36,6 +41,11 @@ public byte[] Get(byte table, byte[] key) return this.db.Get(new[] { table }.Concat(key).ToArray()); } + public byte[] Get(byte[] key) + { + return this.db.Get(key); + } + public void Dispose() { this.db.Dispose(); @@ -54,12 +64,24 @@ public RocksDbBatch(RocksDbSharp.RocksDb db) public IDbBatch Put(byte table, byte[] key, byte[] value) { - return (IDbBatch)this.Put(new[] { table }.Concat(key).ToArray(), value); + return this.Put(new[] { table }.Concat(key).ToArray(), value); } public IDbBatch Delete(byte table, byte[] key) { - return (IDbBatch)this.Delete(new[] { table }.Concat(key).ToArray()); + return this.Delete(new[] { table }.Concat(key).ToArray()); + } + + public IDbBatch Put(byte[] key, byte[] value) + { + base.Put(key, value); + return this; + } + + public IDbBatch Delete(byte[] key) + { + base.Delete(key); + return this; } public void Write() @@ -71,9 +93,14 @@ public void Write() /// A minimal RocksDb wrapper that makes it compliant with the interface. public class RocksDbIterator : IDbIterator { - private byte table; + private byte? table; private Iterator iterator; + public RocksDbIterator(Iterator iterator) + { + this.iterator = iterator; + } + public RocksDbIterator(byte table, Iterator iterator) { this.table = table; @@ -82,11 +109,17 @@ public RocksDbIterator(byte table, Iterator iterator) public void Seek(byte[] key) { - this.iterator.Seek(new[] { this.table }.Concat(key).ToArray()); + this.iterator.Seek(this.table.HasValue ? new[] { this.table.Value }.Concat(key).ToArray() : key); } public void SeekToLast() { + if (!this.table.HasValue) + { + this.iterator.SeekToLast(); + return; + } + if (this.table != 255) { // First seek past the last record in the table by attempting to seek to the start of the next table (if any). @@ -116,12 +149,12 @@ public void Prev() public bool IsValid() { - return this.iterator.Valid() && this.iterator.Value()[0] == this.table; + return this.iterator.Valid() && (!this.table.HasValue || this.iterator.Value()[0] == this.table); } public byte[] Key() { - return this.iterator.Key().Skip(1).ToArray(); + return this.table.HasValue ? this.iterator.Key().Skip(1).ToArray() : this.iterator.Key(); } public byte[] Value() diff --git a/src/Stratis.Bitcoin/Persistence/KeyValueStores/LevelDbKeyValueRepository.cs b/src/Stratis.Bitcoin/Persistence/KeyValueStores/KeyValueRepository.cs similarity index 62% rename from src/Stratis.Bitcoin/Persistence/KeyValueStores/LevelDbKeyValueRepository.cs rename to src/Stratis.Bitcoin/Persistence/KeyValueStores/KeyValueRepository.cs index 53a79e20aa..0967df60e1 100644 --- a/src/Stratis.Bitcoin/Persistence/KeyValueStores/LevelDbKeyValueRepository.cs +++ b/src/Stratis.Bitcoin/Persistence/KeyValueStores/KeyValueRepository.cs @@ -1,47 +1,51 @@ using System.Collections.Generic; using System.IO; using System.Text; -using LevelDB; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Utilities; using Stratis.Bitcoin.Utilities.JsonConverters; namespace Stratis.Bitcoin.Persistence.KeyValueStores { - public class LevelDbKeyValueRepository : IKeyValueRepository + public class KeyValueRepository : IKeyValueRepository where T : IDb, new() { /// Access to database. - private readonly DB leveldb; + private readonly IDb db; private readonly DBreezeSerializer dBreezeSerializer; - public LevelDbKeyValueRepository(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(dataFolder.KeyValueRepositoryPath, dBreezeSerializer) + public KeyValueRepository(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(dataFolder.KeyValueRepositoryPath, dBreezeSerializer) { } - public LevelDbKeyValueRepository(string folder, DBreezeSerializer dBreezeSerializer) + public KeyValueRepository(string folder, DBreezeSerializer dBreezeSerializer) { Directory.CreateDirectory(folder); this.dBreezeSerializer = dBreezeSerializer; // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, folder); + this.db = new T(); + this.db.Open(folder); } /// public void SaveBytes(string key, byte[] bytes, bool overWrite = false) { - byte[] keyBytes = Encoding.ASCII.GetBytes(key); - - if (overWrite) + using (var batch = this.db.GetWriteBatch()) { - byte[] row = this.leveldb.Get(keyBytes); - if (row != null) - this.leveldb.Delete(keyBytes); - } + byte[] keyBytes = Encoding.ASCII.GetBytes(key); - this.leveldb.Put(keyBytes, bytes); + if (overWrite) + { + byte[] row = this.db.Get(keyBytes); + if (row != null) + batch.Delete(keyBytes); + } + + batch.Put(keyBytes, bytes); + batch.Write(); + } } /// @@ -64,7 +68,7 @@ public byte[] LoadBytes(string key) { byte[] keyBytes = Encoding.ASCII.GetBytes(key); - byte[] row = this.leveldb.Get(keyBytes); + byte[] row = this.db.Get(keyBytes); if (row == null) return null; @@ -103,17 +107,17 @@ public T LoadValueJson(string key) public List GetAllAsJson() { var values = new List(); - IEnumerator> enumerator = this.leveldb.GetEnumerator(); - while (enumerator.MoveNext()) + using (var iterator = this.db.GetIterator()) { - (byte[] key, byte[] value) = enumerator.Current; - - if (value == null) - continue; - - string json = Encoding.ASCII.GetString(value); - values.Add(Serializer.ToObject(json)); + foreach ((byte[] key, byte[] value) in iterator.GetAll()) + { + if (value == null) + continue; + + string json = Encoding.ASCII.GetString(value); + values.Add(Serializer.ToObject(json)); + } } return values; @@ -124,15 +128,21 @@ public void Delete(string key) { byte[] keyBytes = Encoding.ASCII.GetBytes(key); - byte[] row = this.leveldb.Get(keyBytes); + byte[] row = this.db.Get(keyBytes); if (row != null) - this.leveldb.Delete(keyBytes); + { + using (var batch = this.db.GetWriteBatch()) + { + batch.Delete(keyBytes); + batch.Write(); + } + } } /// public void Dispose() { - this.leveldb.Dispose(); + this.db.Dispose(); } } } \ No newline at end of file diff --git a/src/Stratis.Bitcoin/Persistence/KeyValueStores/RocksDbKeyValueRepository.cs b/src/Stratis.Bitcoin/Persistence/KeyValueStores/RocksDbKeyValueRepository.cs deleted file mode 100644 index 96547332a7..0000000000 --- a/src/Stratis.Bitcoin/Persistence/KeyValueStores/RocksDbKeyValueRepository.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; -using Stratis.Bitcoin.Utilities.JsonConverters; - -namespace Stratis.Bitcoin.Persistence.KeyValueStores -{ - public class RocksDbKeyValueRepository : IKeyValueRepository - { - private readonly DBreezeSerializer dataStoreSerializer; - private readonly DbOptions dbOptions; - private readonly RocksDb rocksdb; - - public RocksDbKeyValueRepository(DataFolder dataFolder, DBreezeSerializer dataStoreSerializer) - { - Directory.CreateDirectory(dataFolder.KeyValueRepositoryPath); - this.dataStoreSerializer = dataStoreSerializer; - this.dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksdb = RocksDb.Open(this.dbOptions, dataFolder.KeyValueRepositoryPath); - } - - /// - public void SaveBytes(string key, byte[] bytes, bool overWrite = false) - { - byte[] keyBytes = Encoding.ASCII.GetBytes(key); - - if (overWrite) - { - byte[] row = this.rocksdb.Get(keyBytes); - if (row != null) - this.rocksdb.Remove(keyBytes); - } - - this.rocksdb.Put(keyBytes, bytes); - } - - /// - public void SaveValue(string key, T value, bool overWrite = false) - { - this.SaveBytes(key, this.dataStoreSerializer.Serialize(value), overWrite); - } - - /// - public void SaveValueJson(string key, T value, bool overWrite = false) - { - string json = Serializer.ToString(value); - byte[] jsonBytes = Encoding.ASCII.GetBytes(json); - - this.SaveBytes(key, jsonBytes, overWrite); - } - - /// - public byte[] LoadBytes(string key) - { - byte[] keyBytes = Encoding.ASCII.GetBytes(key); - byte[] row = this.rocksdb.Get(keyBytes); - - if (row == null) - return null; - - return row; - } - - /// - public T LoadValue(string key) - { - byte[] bytes = this.LoadBytes(key); - - if (bytes == null) - return default; - - T value = this.dataStoreSerializer.Deserialize(bytes); - return value; - } - - /// - public T LoadValueJson(string key) - { - byte[] bytes = this.LoadBytes(key); - - if (bytes == null) - return default; - - string json = Encoding.ASCII.GetString(bytes); - - T value = Serializer.ToObject(json); - - return value; - } - - /// - public List GetAllAsJson() - { - throw new NotImplementedException(); - } - - /// - public void Delete(string key) - { - throw new NotImplementedException(); - } - - /// - public void Dispose() - { - this.rocksdb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/Stratis.Features.FederatedPeg.Tests/Distribution/RewardClaimerTests.cs b/src/Stratis.Features.FederatedPeg.Tests/Distribution/RewardClaimerTests.cs index 9babc68db3..b3fd7f9ffc 100644 --- a/src/Stratis.Features.FederatedPeg.Tests/Distribution/RewardClaimerTests.cs +++ b/src/Stratis.Features.FederatedPeg.Tests/Distribution/RewardClaimerTests.cs @@ -6,6 +6,7 @@ using Stratis.Bitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Wallet.Interfaces; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Networks; @@ -87,7 +88,7 @@ public RewardClaimerTests() public async Task RewardClaimer_RetrieveSingleDepositsAsync() { DataFolder dataFolder = TestBase.CreateDataFolder(this); - var keyValueRepository = new LevelDbKeyValueRepository(dataFolder, this.dbreezeSerializer); + var keyValueRepository = new KeyValueRepository(dataFolder, this.dbreezeSerializer); // Create a "chain" of 30 blocks. this.blocks = ChainedHeadersHelper.CreateConsecutiveHeadersAndBlocks(30, true, network: this.network, chainIndexer: this.chainIndexer, withCoinbaseAndCoinStake: true, createCirrusReward: true); @@ -115,7 +116,7 @@ public async Task RewardClaimer_RetrieveSingleDepositsAsync() public async Task RewardClaimer_RetrieveBatchedDepositsAsync() { DataFolder dataFolder = TestBase.CreateDataFolder(this); - var keyValueRepository = new LevelDbKeyValueRepository(dataFolder, this.dbreezeSerializer); + var keyValueRepository = new KeyValueRepository(dataFolder, this.dbreezeSerializer); // Create a "chain" of 30 blocks. this.blocks = ChainedHeadersHelper.CreateConsecutiveHeadersAndBlocks(30, true, network: this.network, chainIndexer: this.chainIndexer, withCoinbaseAndCoinStake: true, createCirrusReward: true); diff --git a/src/Stratis.Features.FederatedPeg/Conversion/ReplenishmentKeyValueStore.cs b/src/Stratis.Features.FederatedPeg/Conversion/ReplenishmentKeyValueStore.cs index 97d2868988..b089105a94 100644 --- a/src/Stratis.Features.FederatedPeg/Conversion/ReplenishmentKeyValueStore.cs +++ b/src/Stratis.Features.FederatedPeg/Conversion/ReplenishmentKeyValueStore.cs @@ -2,6 +2,7 @@ using System.Numerics; using Newtonsoft.Json; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Persistence.KeyValueStores; using Stratis.Bitcoin.Utilities; @@ -19,7 +20,7 @@ public interface IReplenishmentKeyValueStore : IKeyValueRepository /// /// Stores wSTRAX replenishment transactions on disk. /// - public sealed class ReplenishmentKeyValueStore : LevelDbKeyValueRepository, IReplenishmentKeyValueStore + public sealed class ReplenishmentKeyValueStore : KeyValueRepository, IReplenishmentKeyValueStore { public ReplenishmentKeyValueStore(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(dataFolder.InteropReplenishmentRepositoryPath, dBreezeSerializer) { diff --git a/src/Stratis.Features.FederatedPeg/Coordination/ConversionRequestFeeKeyValueStore.cs b/src/Stratis.Features.FederatedPeg/Coordination/ConversionRequestFeeKeyValueStore.cs index 01630a89b9..453e8f4a2e 100644 --- a/src/Stratis.Features.FederatedPeg/Coordination/ConversionRequestFeeKeyValueStore.cs +++ b/src/Stratis.Features.FederatedPeg/Coordination/ConversionRequestFeeKeyValueStore.cs @@ -1,4 +1,5 @@ using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Persistence.KeyValueStores; using Stratis.Bitcoin.Utilities; @@ -12,7 +13,7 @@ public interface IConversionRequestFeeKeyValueStore : IKeyValueRepository { } - public sealed class ConversionRequestFeeKeyValueStore : LevelDbKeyValueRepository, IConversionRequestFeeKeyValueStore + public sealed class ConversionRequestFeeKeyValueStore : KeyValueRepository, IConversionRequestFeeKeyValueStore { public ConversionRequestFeeKeyValueStore(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(dataFolder.InteropFeeRepositoryPath, dBreezeSerializer) { From b17790da38f7f485c09f23b59f06a8b511d8e53b Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 8 Aug 2022 16:43:50 +1000 Subject: [PATCH 24/44] Remove redundant CoinDb code (#1021) * Remove redundant CoinDb code * Fix RocksDb.IsValid * Update test * Remove whitespace (cherry picked from commit ae5499ef3c35788bad214a0f54d7fe17972b5681) --- .../CoinViews/CoinviewTests.cs | 2 +- .../CoinViews/CachedCoinView.cs | 2 +- .../CoinViews/Coindb/Coindb.cs | 64 ++++++++----------- .../CoinViews/Coindb/DBreezeCoindb.cs | 2 +- .../CoinViews/Coindb/ICoindb.cs | 3 +- .../CoinViews/InMemoryCoinView.cs | 2 +- .../CoinViewTests.cs | 2 +- .../NodeContext.cs | 6 +- 8 files changed, 37 insertions(+), 46 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 8f882403a4..4d71c6b4ff 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -43,7 +43,7 @@ public CoinviewTests() this.nodeStats = new NodeStats(this.dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object); this.coindb = new DBreezeCoindb(this.network, this.dataFolder, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory)); - this.coindb.Initialize(new ChainedHeader(this.network.GetGenesis().Header, this.network.GenesisHash, 0)); + this.coindb.Initialize(); this.chainIndexer = new ChainIndexer(this.network); this.stakeChainStore = new StakeChainStore(this.network, this.chainIndexer, (IStakedb)this.coindb, this.loggerFactory); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index e77ad63bb0..86b5279c00 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -200,7 +200,7 @@ public void Sync(ChainIndexer chainIndexer) public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) { - this.coindb.Initialize(chainTip); + this.coindb.Initialize(); Sync(chainIndexer); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index be0cd8603a..64c27140be 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -41,7 +41,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private BackendPerformanceSnapshot latestPerformanceSnapShot; - /// Access to dBreeze database. + /// Access to database. private IDb coinDb; private readonly DBreezeSerializer dBreezeSerializer; @@ -70,12 +70,32 @@ public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProv nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); } - public void Initialize(ChainedHeader chainTip) + public void Initialize() { // Open a connection to a new DB and create if not found this.coinDb = new T(); this.coinDb.Open(this.dataFolder); + EndiannessFix(); + + EnsureCoinDatabaseIntegrity(); + + Block genesis = this.network.GetGenesis(); + + if (this.GetTipHash() == null) + { + using (var batch = this.coinDb.GetWriteBatch()) + { + this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); + batch.Write(); + } + } + + this.logger.LogInformation("Coin database initialized with tip '{0}'.", this.persistedCoinviewTip); + } + + private void EndiannessFix() + { // Check if key bytes are in the wrong endian order. HashHeightPair current = this.GetTipHash(); @@ -129,46 +149,18 @@ public void Initialize(ChainedHeader chainTip) } } } - - EnsureCoinDatabaseIntegrity(chainTip); - - Block genesis = this.network.GetGenesis(); - - if (this.GetTipHash() == null) - { - using (var batch = this.coinDb.GetWriteBatch()) - { - this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); - batch.Write(); - } - } - - this.logger.LogInformation("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip); } - private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) + /// Just check the integrity. Coin view performs the sync with the chain tip. + private void EnsureCoinDatabaseIntegrity() { this.logger.LogInformation("Checking coin database integrity..."); - var heightToCheck = chainTip.Height; - - // Find the height up to where rewind data is stored above chain tip. - do - { - heightToCheck += 1; - - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); - if (row == null) - break; - - } while (true); - - for (int height = heightToCheck - 1; height > chainTip.Height;) + if (this.GetTipHash() == null) { - this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{chainTip}'."); - - // Do a batch of rewinding. - height = RewindInternal(height, new HashHeightPair(chainTip)).Height; + this.logger.LogInformation($"Rebuilding coin database that has no tip information."); + this.coinDb.Clear(); + return; } this.logger.LogInformation("Coin database integrity good."); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs index 638444e332..0864d972a0 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs @@ -65,7 +65,7 @@ public DBreezeCoindb(Network network, string folder, IDateTimeProvider dateTimeP nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300); } - public void Initialize(ChainedHeader chainTip) + public void Initialize() { Block genesis = this.network.GetGenesis(); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index f8c664d6b6..076f696aed 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -10,8 +10,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews public interface ICoindb { /// Initialize the coin database. - /// The current chain's tip. - void Initialize(ChainedHeader chainTip); + void Initialize(); /// /// Retrieves the block hash of the current tip of the coinview. diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 9cb80e7129..7652b469a1 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -122,7 +122,7 @@ public RewindData GetRewindData(int height) throw new NotImplementedException(); } - public void Initialize(ChainedHeader chainTip = null) + public void Initialize() { } } diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index eed6363a16..283d30207e 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -56,7 +56,7 @@ public void TestDBreezeSerialization() chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); ctx.Coindb.SaveChanges(new List(), new HashHeightPair(previous), new HashHeightPair(chained)); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); - ctx.ReloadPersistentCoinView(chained); + ctx.ReloadPersistentCoinView(); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); Assert.NotNull(ctx.Coindb.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); Assert.Null(ctx.Coindb.FetchCoins(new[] { new OutPoint() }).UnspentOutputs.Values.FirstOrDefault().Coins); diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index fb005f0b45..b503f3ea3d 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -29,7 +29,7 @@ public NodeContext(object caller, string name, Network network) var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); this.Coindb = new Coindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); - this.Coindb.Initialize(new ChainedHeader(network.GetGenesis().Header, network.GenesisHash, 0)); + this.Coindb.Initialize(); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -60,7 +60,7 @@ public void Dispose() item.Dispose(); } - public void ReloadPersistentCoinView(ChainedHeader chainTip) + public void ReloadPersistentCoinView() { ((IDisposable)this.Coindb).Dispose(); this.cleanList.Remove((IDisposable)this.Coindb); @@ -68,7 +68,7 @@ public void ReloadPersistentCoinView(ChainedHeader chainTip) var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); this.Coindb = new Coindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); - this.Coindb.Initialize(chainTip); + this.Coindb.Initialize(); this.cleanList.Add((IDisposable)this.Coindb); } } From 7954a393f2315a2d431bb133fbee393a733688ed Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 8 Aug 2022 16:44:07 +1000 Subject: [PATCH 25/44] Combine BlockRepository implementations (#1030) * Combine KeyValueStore implementations * Update tests * Combine BlockRepository implementations * RocksDb fix * Update tests * Remove unused method (cherry picked from commit 5a62c21a1ffdcb23ad2534a799395f43bc71e67e) --- .../BlockRepositoryTests.cs | 3 +- .../PruneBlockRepositoryTests.cs | 7 +- .../BlockStoreFeature.cs | 5 +- ...bBlockRepository.cs => BlockRepository.cs} | 144 ++--- .../Repositories/RocksDbBlockRepository.cs | 556 ------------------ .../TestChainFactory.cs | 3 +- .../CoinViews/Coindb/Coindb.cs | 4 +- .../BlockStore/BlockStoreTests.cs | 3 +- .../Database/IDbIteratorExt.cs | 13 +- src/Stratis.Bitcoin/Database/RocksDb.cs | 2 +- .../WalletRepositoryTests.cs | 26 +- 11 files changed, 122 insertions(+), 644 deletions(-) rename src/Stratis.Bitcoin.Features.BlockStore/Repositories/{LevelDbBlockRepository.cs => BlockRepository.cs} (76%) delete mode 100644 src/Stratis.Bitcoin.Features.BlockStore/Repositories/RocksDbBlockRepository.cs diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockRepositoryTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockRepositoryTests.cs index 9f22ab1f3b..37dc9d9e18 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockRepositoryTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockRepositoryTests.cs @@ -3,6 +3,7 @@ using System.Linq; using LevelDB; using NBitcoin; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore.Repositories; using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Tests.Common.Logging; @@ -537,7 +538,7 @@ public void GetBlockIdByTransactionIdForGenesisBlock() private IBlockRepository SetupRepository(Network main, string dataFolder) { var dBreezeSerializer = new DBreezeSerializer(main.Consensus.ConsensusFactory); - var repository = new LevelDbBlockRepository(main, dataFolder, dBreezeSerializer); + var repository = new BlockRepository(main, dataFolder, dBreezeSerializer); repository.Initialize(); return repository; } diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/PruneBlockRepositoryTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/PruneBlockRepositoryTests.cs index fd8406c1e4..eaa021689d 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/PruneBlockRepositoryTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/PruneBlockRepositoryTests.cs @@ -1,5 +1,6 @@ using System.Linq; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore.Pruning; using Stratis.Bitcoin.Features.BlockStore.Repositories; using Stratis.Bitcoin.Networks; @@ -26,7 +27,7 @@ public void PruneRepository_PruneAndCompact_FromGenesis_OnStartUp() var dBreezeSerializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - var blockRepository = new LevelDbBlockRepository(this.Network, dataFolder, dBreezeSerializer); + var blockRepository = new BlockRepository(this.Network, dataFolder, dBreezeSerializer); blockRepository.Initialize(); blockRepository.PutBlocks(new HashHeightPair(posBlocks.Last().GetHash(), 50), posBlocks); @@ -57,7 +58,7 @@ public void PruneRepository_PruneAndCompact_MidChain_OnStartUp() var dBreezeSerializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - var blockRepository = new LevelDbBlockRepository(this.Network, dataFolder, dBreezeSerializer); + var blockRepository = new BlockRepository(this.Network, dataFolder, dBreezeSerializer); blockRepository.Initialize(); blockRepository.PutBlocks(new HashHeightPair(posBlocks.Take(100).Last().GetHash(), 100), posBlocks.Take(100).ToList()); @@ -97,7 +98,7 @@ public void PruneRepository_PruneAndCompact_OnShutDown() var dBreezeSerializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - var blockRepository = new LevelDbBlockRepository(this.Network, dataFolder, dBreezeSerializer); + var blockRepository = new BlockRepository(this.Network, dataFolder, dBreezeSerializer); blockRepository.Initialize(); blockRepository.PutBlocks(new HashHeightPair(posBlocks.Last().GetHash(), 50), posBlocks); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs index 10d2f550eb..7fc5d0a9d6 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs @@ -10,6 +10,7 @@ using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Connection; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.BlockStore.Pruning; using Stratis.Bitcoin.Features.BlockStore.Repositories; @@ -185,10 +186,10 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild services.AddSingleton().AddSingleton(provider => provider.GetService()); if (dbType == DbType.Leveldb) - services.AddSingleton(); + services.AddSingleton>(); if (dbType == DbType.RocksDb) - services.AddSingleton(); + services.AddSingleton>(); services.AddSingleton(); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/LevelDbBlockRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs similarity index 76% rename from src/Stratis.Bitcoin.Features.BlockStore/Repositories/LevelDbBlockRepository.cs rename to src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs index 52dc357c29..0aa1a8ad99 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/LevelDbBlockRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs @@ -5,21 +5,19 @@ using System.Text; using System.Threading; using DBreeze.Utils; -using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Persistence; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.BlockStore.Repositories { - public class LevelDbBlockRepository : IBlockRepository + public class BlockRepository : IBlockRepository where T : IDb, new() { private readonly string dataFolder; - private Options dbOptions; - private DB leveldb; + private IDb db; private readonly object locker; private readonly ILogger logger; private readonly Network network; @@ -33,18 +31,15 @@ public class LevelDbBlockRepository : IBlockRepository /// public bool TxIndex { get; private set; } - - public DB Leveldb => this.leveldb; - private readonly DBreezeSerializer dBreezeSerializer; private IReadOnlyDictionary genesisTransactions; - public LevelDbBlockRepository(Network network, DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) + public BlockRepository(Network network, DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(network, dataFolder.BlockPath, dBreezeSerializer) { } - public LevelDbBlockRepository(Network network, string dataFolder, DBreezeSerializer dBreezeSerializer) + public BlockRepository(Network network, string dataFolder, DBreezeSerializer dBreezeSerializer) { this.dataFolder = dataFolder; this.locker = new object(); @@ -59,8 +54,8 @@ public void Initialize() { Directory.CreateDirectory(this.dataFolder); - this.dbOptions = new Options { CreateIfMissing = true }; - this.leveldb = new DB(this.dbOptions, this.dataFolder); + this.db = new T(); + this.db.Open(this.dataFolder); Block genesis = this.network.GetGenesis(); this.genesisTransactions = this.network.GetGenesis().Transactions.ToDictionary(k => k.GetHash()); @@ -98,7 +93,7 @@ public Transaction GetTransactionById(uint256 trxid) Transaction res = null; lock (this.locker) { - byte[] transactionRow = this.leveldb.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); + byte[] transactionRow = this.db.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); if (transactionRow == null) { @@ -106,7 +101,7 @@ public Transaction GetTransactionById(uint256 trxid) return null; } - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, transactionRow); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, transactionRow); if (blockRow != null) { @@ -151,14 +146,14 @@ public Transaction GetTransactionById(uint256 trxid) continue; } - byte[] transactionRow = this.leveldb.Get(BlockRepositoryConstants.TransactionTableName, trxids[i].ToBytes()); + byte[] transactionRow = this.db.Get(BlockRepositoryConstants.TransactionTableName, trxids[i].ToBytes()); if (transactionRow == null) { this.logger.LogTrace("(-)[NO_TX_ROW]:null"); return null; } - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, transactionRow); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, transactionRow); if (blockRow != null) { @@ -195,7 +190,7 @@ public uint256 GetBlockIdByTransactionId(uint256 trxid) uint256 res = null; lock (this.locker) { - byte[] transactionRow = this.leveldb.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); + byte[] transactionRow = this.db.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); if (transactionRow != null) res = new uint256(transactionRow); } @@ -220,7 +215,7 @@ protected virtual void OnInsertBlocks(List blocks) List> blockList = blockDict.ToList(); blockList.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Key.ToBytes(), pair2.Key.ToBytes())); - using (var batch = new WriteBatch()) + using (var batch = this.db.GetWriteBatch()) { // Index blocks. foreach (KeyValuePair kv in blockList) @@ -229,7 +224,7 @@ protected virtual void OnInsertBlocks(List blocks) Block block = kv.Value; // If the block is already in store don't write it again. - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, blockId.ToBytes()); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, blockId.ToBytes()); if (blockRow == null) { batch.Put(BlockRepositoryConstants.BlockTableName, blockId.ToBytes(), this.dBreezeSerializer.Serialize(block)); @@ -242,7 +237,7 @@ protected virtual void OnInsertBlocks(List blocks) } } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } if (this.TxIndex) @@ -254,13 +249,13 @@ protected virtual void OnInsertTransactions(List<(Transaction, Block)> transacti var byteListComparer = new ByteListComparer(); transactions.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Item1.GetHash().ToBytes(), pair2.Item1.GetHash().ToBytes())); - using (var batch = new WriteBatch()) + using (var batch = this.db.GetWriteBatch()) { // Index transactions. foreach ((Transaction transaction, Block block) in transactions) batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -271,7 +266,7 @@ public IEnumerable EnumerateBatch(List headers) { foreach (ChainedHeader chainedHeader in headers) { - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, chainedHeader.HashBlock.ToBytes()); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, chainedHeader.HashBlock.ToBytes()); Block block = blockRow != null ? this.dBreezeSerializer.Deserialize(blockRow) : null; yield return block; } @@ -300,40 +295,42 @@ public void ReIndex() warningMessage.AppendLine(); this.logger.LogInformation(warningMessage.ToString()); - using (var batch = new WriteBatch()) + using (var batch = this.db.GetWriteBatch()) { - var enumerator = this.leveldb.GetEnumerator(); - while (enumerator.MoveNext()) + var enumerator = this.db.GetIterator(BlockRepositoryConstants.BlockTableName); + + foreach ((byte[] key, byte[] value) in enumerator.GetAll()) { - if (enumerator.Current.Key[0] == BlockRepositoryConstants.BlockTableName) + var block = this.dBreezeSerializer.Deserialize(value); + foreach (Transaction transaction in block.Transactions) + { + batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); + } + + // inform the user about the ongoing operation + if (++rowCount % 1000 == 0) { - var block = this.dBreezeSerializer.Deserialize(enumerator.Current.Value); - foreach (Transaction transaction in block.Transactions) - { - batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - } - - // inform the user about the ongoing operation - if (++rowCount % 1000 == 0) - { - this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); - } + this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); } } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } this.logger.LogInformation("Reindex completed successfully."); } else { - var enumerator = this.leveldb.GetEnumerator(); - while (enumerator.MoveNext()) + using (var batch = this.db.GetWriteBatch()) { - // Clear tx from database. - if (enumerator.Current.Key[0] == BlockRepositoryConstants.TransactionTableName) - this.leveldb.Delete(enumerator.Current.Key); + var enumerator = this.db.GetIterator(BlockRepositoryConstants.TransactionTableName); + foreach ((byte[] key, _) in enumerator.GetAll(keysOnly: true)) + { + // Clear tx from database. + batch.Delete(BlockRepositoryConstants.TransactionTableName, key); + } + + batch.Write(); } } } @@ -359,7 +356,7 @@ public void PutBlocks(HashHeightPair newTip, List blocks) private bool? LoadTxIndex() { bool? res = null; - byte[] row = this.leveldb.Get(BlockRepositoryConstants.CommonTableName, TxIndexKey); + byte[] row = this.db.Get(BlockRepositoryConstants.CommonTableName, TxIndexKey); if (row != null) { this.TxIndex = BitConverter.ToBoolean(row, 0); @@ -371,8 +368,12 @@ public void PutBlocks(HashHeightPair newTip, List blocks) private void SaveTxIndex(bool txIndex) { - this.TxIndex = txIndex; - this.leveldb.Put(BlockRepositoryConstants.CommonTableName, TxIndexKey, BitConverter.GetBytes(txIndex)); + using (var batch = this.db.GetWriteBatch()) + { + this.TxIndex = txIndex; + batch.Put(BlockRepositoryConstants.CommonTableName, TxIndexKey, BitConverter.GetBytes(txIndex)); + batch.Write(); + } } /// @@ -388,7 +389,7 @@ private HashHeightPair LoadTipHashAndHeight() { if (this.TipHashAndHeight == null) { - byte[] row = this.leveldb.Get(BlockRepositoryConstants.CommonTableName, RepositoryTipKey); + byte[] row = this.db.Get(BlockRepositoryConstants.CommonTableName, RepositoryTipKey); if (row != null) this.TipHashAndHeight = this.dBreezeSerializer.Deserialize(row); } @@ -398,8 +399,12 @@ private HashHeightPair LoadTipHashAndHeight() private void SaveTipHashAndHeight(HashHeightPair newTip) { - this.TipHashAndHeight = newTip; - this.leveldb.Put(BlockRepositoryConstants.CommonTableName, RepositoryTipKey, this.dBreezeSerializer.Serialize(newTip)); + using (var batch = this.db.GetWriteBatch()) + { + this.TipHashAndHeight = newTip; + batch.Put(BlockRepositoryConstants.CommonTableName, RepositoryTipKey, this.dBreezeSerializer.Serialize(newTip)); + batch.Write(); + } } /// @@ -444,7 +449,7 @@ public bool Exist(uint256 hash) { // Lazy loading is on so we don't fetch the whole value, just the row. byte[] key = hash.ToBytes(); - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, key); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, key); if (blockRow != null) res = true; } @@ -452,27 +457,24 @@ public bool Exist(uint256 hash) return res; } - protected virtual void OnDeleteTransactions(List<(Transaction, Block)> transactions) - { - foreach ((Transaction transaction, Block block) in transactions) - this.leveldb.Delete(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes()); - } - protected virtual void OnDeleteBlocks(List blocks) { - if (this.TxIndex) + using (var batch = this.db.GetWriteBatch()) { - var transactions = new List<(Transaction, Block)>(); + if (this.TxIndex) + { + var transactions = new List<(Transaction, Block)>(); + + foreach (Block block in blocks) + foreach (Transaction transaction in block.Transactions) + batch.Delete(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes()); + } foreach (Block block in blocks) - foreach (Transaction transaction in block.Transactions) - transactions.Add((transaction, block)); + batch.Delete(BlockRepositoryConstants.BlockTableName, block.GetHash().ToBytes()); - this.OnDeleteTransactions(transactions); + batch.Write(); } - - foreach (Block block in blocks) - this.leveldb.Delete(BlockRepositoryConstants.BlockTableName, block.GetHash().ToBytes()); } public List GetBlocksFromHashes(List hashes) @@ -494,7 +496,7 @@ public List GetBlocksFromHashes(List hashes) continue; } - byte[] blockRow = this.leveldb.Get(BlockRepositoryConstants.BlockTableName, key.Item2); + byte[] blockRow = this.db.Get(BlockRepositoryConstants.BlockTableName, key.Item2); if (blockRow != null) { results[key.Item1] = this.dBreezeSerializer.Deserialize(blockRow); @@ -542,19 +544,23 @@ public void DeleteBlocks(List hashes) public byte[] Get(byte tableName, byte[] key) { - byte[] result = this.leveldb.Get(tableName, key); + byte[] result = this.db.Get(tableName, key); return result; } public void Put(byte tableName, byte[] key, byte[] value) { - this.leveldb.Put(tableName, key, value); + using (var batch = this.db.GetWriteBatch()) + { + batch.Put(tableName, key, value); + batch.Write(); + } } /// public void Dispose() { - this.leveldb.Dispose(); + this.db.Dispose(); } } } diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/RocksDbBlockRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/Repositories/RocksDbBlockRepository.cs deleted file mode 100644 index 4b9fc03490..0000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/RocksDbBlockRepository.cs +++ /dev/null @@ -1,556 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using DBreeze.Utils; -using Microsoft.Extensions.Logging; -using NBitcoin; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Persistence; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.BlockStore.Repositories -{ - public class RocksDbBlockRepository : IBlockRepository - { - private readonly string dataFolder; - private DbOptions dbOptions; - private RocksDb rocksDb; - private readonly object locker; - private readonly ILogger logger; - private readonly Network network; - - private static readonly byte[] RepositoryTipKey = new byte[0]; - - private static readonly byte[] TxIndexKey = new byte[1]; - - /// - public HashHeightPair TipHashAndHeight { get; private set; } - - /// - public bool TxIndex { get; private set; } - - private readonly DBreezeSerializer dBreezeSerializer; - private IReadOnlyDictionary genesisTransactions; - - public RocksDbBlockRepository(Network network, string dataFolder, DBreezeSerializer dataStoreSerializer) - { - this.dataFolder = dataFolder; - this.dBreezeSerializer = dataStoreSerializer; - this.locker = new object(); - this.logger = LogManager.GetCurrentClassLogger(); - this.network = network; - } - - public RocksDbBlockRepository(Network network, DataFolder dataFolder, DBreezeSerializer dataStoreSerializer) - : this(network, dataFolder.BlockPath, dataStoreSerializer) - { - } - - /// - public void Initialize() - { - Directory.CreateDirectory(this.dataFolder); - - this.dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder); - - Block genesis = this.network.GetGenesis(); - this.genesisTransactions = this.network.GetGenesis().Transactions.ToDictionary(k => k.GetHash()); - - lock (this.locker) - { - if (this.LoadTipHashAndHeight() == null) - { - this.SaveTipHashAndHeight(new HashHeightPair(genesis.GetHash(), 0)); - } - - if (this.LoadTxIndex() == null) - { - this.SaveTxIndex(false); - } - } - } - - /// - public Transaction GetTransactionById(uint256 trxid) - { - Guard.NotNull(trxid, nameof(trxid)); - - if (!this.TxIndex) - { - this.logger.LogTrace("(-)[TX_INDEXING_DISABLED]:null"); - return default; - } - - if (this.genesisTransactions.TryGetValue(trxid, out Transaction genesisTransaction)) - { - return genesisTransaction; - } - - Transaction res = null; - lock (this.locker) - { - byte[] transactionRow = this.rocksDb.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); - - if (transactionRow == null) - { - this.logger.LogTrace("(-)[NO_BLOCK]:null"); - return null; - } - - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, transactionRow); - - if (blockRow != null) - { - var block = this.dBreezeSerializer.Deserialize(blockRow); - res = block.Transactions.FirstOrDefault(t => t.GetHash() == trxid); - } - } - - return res; - } - - /// - public Transaction[] GetTransactionsByIds(uint256[] trxids, CancellationToken cancellation = default(CancellationToken)) - { - if (!this.TxIndex) - { - this.logger.LogTrace("(-)[TX_INDEXING_DISABLED]:null"); - return null; - } - - Transaction[] txes = new Transaction[trxids.Length]; - - lock (this.locker) - { - for (int i = 0; i < trxids.Length; i++) - { - cancellation.ThrowIfCancellationRequested(); - - bool alreadyFetched = trxids.Take(i).Any(x => x == trxids[i]); - - if (alreadyFetched) - { - this.logger.LogDebug("Duplicated transaction encountered. Tx id: '{0}'.", trxids[i]); - - txes[i] = txes.First(x => x.GetHash() == trxids[i]); - continue; - } - - if (this.genesisTransactions.TryGetValue(trxids[i], out Transaction genesisTransaction)) - { - txes[i] = genesisTransaction; - continue; - } - - byte[] transactionRow = this.rocksDb.Get(BlockRepositoryConstants.TransactionTableName, trxids[i].ToBytes()); - if (transactionRow == null) - { - this.logger.LogTrace("(-)[NO_TX_ROW]:null"); - return null; - } - - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, transactionRow); - - if (blockRow != null) - { - this.logger.LogTrace("(-)[NO_BLOCK]:null"); - return null; - } - - var block = this.dBreezeSerializer.Deserialize(blockRow); - Transaction tx = block.Transactions.FirstOrDefault(t => t.GetHash() == trxids[i]); - - txes[i] = tx; - } - } - - return txes; - } - - /// - public uint256 GetBlockIdByTransactionId(uint256 trxid) - { - Guard.NotNull(trxid, nameof(trxid)); - - if (!this.TxIndex) - { - this.logger.LogTrace("(-)[NO_TXINDEX]:null"); - return default; - } - - if (this.genesisTransactions.ContainsKey(trxid)) - { - return this.network.GenesisHash; - } - - uint256 res = null; - lock (this.locker) - { - byte[] transactionRow = this.rocksDb.Get(BlockRepositoryConstants.TransactionTableName, trxid.ToBytes()); - if (transactionRow != null) - res = new uint256(transactionRow); - } - - return res; - } - - protected virtual void OnInsertBlocks(List blocks) - { - var transactions = new List<(Transaction, Block)>(); - var byteListComparer = new ByteListComparer(); - var blockDict = new Dictionary(); - - // Gather blocks. - foreach (Block block in blocks) - { - uint256 blockId = block.GetHash(); - blockDict[blockId] = block; - } - - // Sort blocks. Be consistent in always converting our keys to byte arrays using the ToBytes method. - List> blockList = blockDict.ToList(); - blockList.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Key.ToBytes(), pair2.Key.ToBytes())); - - using (var batch = new WriteBatch()) - { - // Index blocks. - foreach (KeyValuePair kv in blockList) - { - uint256 blockId = kv.Key; - Block block = kv.Value; - - // If the block is already in store don't write it again. - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, blockId.ToBytes()); - if (blockRow == null) - { - batch.Put(BlockRepositoryConstants.BlockTableName, blockId.ToBytes(), this.dBreezeSerializer.Serialize(block)); - - if (this.TxIndex) - { - foreach (Transaction transaction in block.Transactions) - transactions.Add((transaction, block)); - } - } - } - - this.rocksDb.Write(batch); - } - - if (this.TxIndex) - this.OnInsertTransactions(transactions); - } - - protected virtual void OnInsertTransactions(List<(Transaction, Block)> transactions) - { - var byteListComparer = new ByteListComparer(); - transactions.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Item1.GetHash().ToBytes(), pair2.Item1.GetHash().ToBytes())); - - using (var batch = new WriteBatch()) - { - // Index transactions. - foreach ((Transaction transaction, Block block) in transactions) - batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - - this.rocksDb.Write(batch); - } - } - - public IEnumerable EnumerateBatch(List headers) - { - lock (this.locker) - { - foreach (ChainedHeader chainedHeader in headers) - { - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, chainedHeader.HashBlock.ToBytes()); - Block block = blockRow != null ? this.dBreezeSerializer.Deserialize(blockRow) : null; - yield return block; - } - } - } - - /// - public void ReIndex() - { - lock (this.locker) - { - if (this.TxIndex) - { - int rowCount = 0; - // Insert transactions to database. - - int totalBlocksCount = this.TipHashAndHeight?.Height ?? 0; - - var warningMessage = new StringBuilder(); - warningMessage.AppendLine("".PadRight(59, '=') + " W A R N I N G " + "".PadRight(59, '=')); - warningMessage.AppendLine(); - warningMessage.AppendLine($"Starting ReIndex process on a total of {totalBlocksCount} blocks."); - warningMessage.AppendLine("The operation could take a long time, please don't stop it."); - warningMessage.AppendLine(); - warningMessage.AppendLine("".PadRight(133, '=')); - warningMessage.AppendLine(); - - this.logger.LogInformation(warningMessage.ToString()); - using (var batch = new WriteBatch()) - { - var enumerator = this.rocksDb.NewIterator(); - for (enumerator.SeekToFirst(); enumerator.Valid(); enumerator.Next()) - { - if (enumerator.Key()[0] == BlockRepositoryConstants.BlockTableName) - { - var block = this.dBreezeSerializer.Deserialize(enumerator.Value()); - foreach (Transaction transaction in block.Transactions) - { - batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - } - - // inform the user about the ongoing operation - if (++rowCount % 1000 == 0) - { - this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); - } - } - } - - this.rocksDb.Write(batch); - } - - this.logger.LogInformation("Reindex completed successfully."); - } - else - { - var enumerator = this.rocksDb.NewIterator(); - for (enumerator.SeekToFirst(); enumerator.Valid(); enumerator.Next()) - { - // Clear tx from database. - if (enumerator.Key()[0] == BlockRepositoryConstants.TransactionTableName) - this.rocksDb.Remove(enumerator.Key()); - } - } - } - } - - /// - public void PutBlocks(HashHeightPair newTip, List blocks) - { - Guard.NotNull(newTip, nameof(newTip)); - Guard.NotNull(blocks, nameof(blocks)); - - // DBreeze is faster if sort ascending by key in memory before insert - // however we need to find how byte arrays are sorted in DBreeze. - lock (this.locker) - { - this.OnInsertBlocks(blocks); - - // Commit additions - this.SaveTipHashAndHeight(newTip); - } - } - - private bool? LoadTxIndex() - { - bool? res = null; - byte[] row = this.rocksDb.Get(BlockRepositoryConstants.CommonTableName, TxIndexKey); - if (row != null) - { - this.TxIndex = BitConverter.ToBoolean(row); - res = this.TxIndex; - } - - return res; - } - - private void SaveTxIndex(bool txIndex) - { - this.TxIndex = txIndex; - this.rocksDb.Put(BlockRepositoryConstants.CommonTableName, TxIndexKey, BitConverter.GetBytes(txIndex)); - } - - /// - public void SetTxIndex(bool txIndex) - { - lock (this.locker) - { - this.SaveTxIndex(txIndex); - } - } - - private HashHeightPair LoadTipHashAndHeight() - { - if (this.TipHashAndHeight == null) - { - byte[] row = this.rocksDb.Get(BlockRepositoryConstants.CommonTableName, RepositoryTipKey); - if (row != null) - this.TipHashAndHeight = this.dBreezeSerializer.Deserialize(row); - } - - return this.TipHashAndHeight; - } - - private void SaveTipHashAndHeight(HashHeightPair newTip) - { - this.TipHashAndHeight = newTip; - this.rocksDb.Put(BlockRepositoryConstants.CommonTableName, RepositoryTipKey, this.dBreezeSerializer.Serialize(newTip)); - } - - /// - public Block GetBlock(uint256 hash) - { - Guard.NotNull(hash, nameof(hash)); - - Block res = null; - lock (this.locker) - { - var results = this.GetBlocksFromHashes(new List { hash }); - - if (results.FirstOrDefault() != null) - res = results.FirstOrDefault(); - } - - return res; - } - - /// - public List GetBlocks(List hashes) - { - Guard.NotNull(hashes, nameof(hashes)); - - List blocks; - - lock (this.locker) - { - blocks = this.GetBlocksFromHashes(hashes); - } - - return blocks; - } - - /// - public bool Exist(uint256 hash) - { - Guard.NotNull(hash, nameof(hash)); - - bool res = false; - lock (this.locker) - { - // Lazy loading is on so we don't fetch the whole value, just the row. - byte[] key = hash.ToBytes(); - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, key); - if (blockRow != null) - res = true; - } - - return res; - } - - protected virtual void OnDeleteTransactions(List<(Transaction, Block)> transactions) - { - foreach ((Transaction transaction, Block block) in transactions) - this.rocksDb.Remove(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes()); - } - - protected virtual void OnDeleteBlocks(List blocks) - { - if (this.TxIndex) - { - var transactions = new List<(Transaction, Block)>(); - - foreach (Block block in blocks) - foreach (Transaction transaction in block.Transactions) - transactions.Add((transaction, block)); - - this.OnDeleteTransactions(transactions); - } - - foreach (Block block in blocks) - this.rocksDb.Remove(BlockRepositoryConstants.BlockTableName, block.GetHash().ToBytes()); - } - - public List GetBlocksFromHashes(List hashes) - { - var results = new Dictionary(); - - // Access hash keys in sorted order. - var byteListComparer = new ByteListComparer(); - List<(uint256, byte[])> keys = hashes.Select(hash => (hash, hash.ToBytes())).ToList(); - - keys.Sort((key1, key2) => byteListComparer.Compare(key1.Item2, key2.Item2)); - - foreach ((uint256, byte[]) key in keys) - { - // If searching for genesis block, return it. - if (key.Item1 == this.network.GenesisHash) - { - results[key.Item1] = this.network.GetGenesis(); - continue; - } - - byte[] blockRow = this.rocksDb.Get(BlockRepositoryConstants.BlockTableName, key.Item2); - if (blockRow != null) - { - results[key.Item1] = this.dBreezeSerializer.Deserialize(blockRow); - - this.logger.LogDebug("Block hash '{0}' loaded from the store.", key.Item1); - } - else - { - results[key.Item1] = null; - - this.logger.LogDebug("Block hash '{0}' not found in the store.", key.Item1); - } - } - - // Return the result in the order that the hashes were presented. - return hashes.Select(hash => results[hash]).ToList(); - } - - /// - public void Delete(HashHeightPair newTip, List hashes) - { - Guard.NotNull(newTip, nameof(newTip)); - Guard.NotNull(hashes, nameof(hashes)); - - lock (this.locker) - { - List blocks = this.GetBlocksFromHashes(hashes); - this.OnDeleteBlocks(blocks.Where(b => b != null).ToList()); - this.SaveTipHashAndHeight(newTip); - } - } - - /// - public void DeleteBlocks(List hashes) - { - Guard.NotNull(hashes, nameof(hashes)); - - lock (this.locker) - { - List blocks = this.GetBlocksFromHashes(hashes); - - this.OnDeleteBlocks(blocks.Where(b => b != null).ToList()); - } - } - - public byte[] Get(byte tableName, byte[] key) - { - byte[] result = this.rocksDb.Get(tableName, key); - return result; - } - - public void Put(byte tableName, byte[] key, byte[] value) - { - this.rocksDb.Put(tableName, key, value); - } - - /// - public void Dispose() - { - this.rocksDb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index cd152c776e..b0a2e528e1 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -14,6 +14,7 @@ using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Consensus.Rules; using Stratis.Bitcoin.Consensus.Validators; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore; using Stratis.Bitcoin.Features.BlockStore.Repositories; using Stratis.Bitcoin.Features.Consensus.CoinViews; @@ -153,7 +154,7 @@ public static async Task CreateAsync(Network network, string d var dBreezeSerializer = new DBreezeSerializer(network.Consensus.ConsensusFactory); - var blockRepository = new LevelDbBlockRepository(testChainContext.Network, dataFolder, dBreezeSerializer); + var blockRepository = new BlockRepository(testChainContext.Network, dataFolder, dBreezeSerializer); var blockStoreFlushCondition = new BlockStoreQueueFlushCondition(testChainContext.ChainState, testChainContext.InitialBlockDownloadState); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index 64c27140be..d7a371a8c6 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -115,7 +115,7 @@ private void EndiannessFix() using (var iterator = this.coinDb.GetIterator(rewindTable)) { - iterator.Seek(new byte[0]); + iterator.SeekFirst(); while (iterator.IsValid()) { @@ -277,7 +277,7 @@ public int GetMinRewindHeight() // Find the first row with a rewind table key prefix. using (var iterator = this.coinDb.GetIterator(rewindTable)) { - iterator.Seek(new byte[0]); + iterator.SeekFirst(); if (!iterator.IsValid()) return -1; diff --git a/src/Stratis.Bitcoin.IntegrationTests/BlockStore/BlockStoreTests.cs b/src/Stratis.Bitcoin.IntegrationTests/BlockStore/BlockStoreTests.cs index 4f3f82ef68..ac2b207ce1 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/BlockStore/BlockStoreTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/BlockStore/BlockStoreTests.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using NBitcoin; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore; using Stratis.Bitcoin.Features.BlockStore.Repositories; using Stratis.Bitcoin.IntegrationTests.Common; @@ -27,7 +28,7 @@ public void BlockRepositoryPutBatch() { var dBreezeSerializer = new DBreezeSerializer(this.network.Consensus.ConsensusFactory); - using (var blockRepository = new LevelDbBlockRepository(this.network, TestBase.CreateDataFolder(this), dBreezeSerializer)) + using (var blockRepository = new BlockRepository(this.network, TestBase.CreateDataFolder(this), dBreezeSerializer)) { blockRepository.Initialize(); diff --git a/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs index 35c4796649..d4962ffbe4 100644 --- a/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs +++ b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs @@ -11,10 +11,19 @@ public static class IDbIteratorExt { private static ByteArrayComparer byteArrayComparer = new ByteArrayComparer(); + /// + /// Seeks to the first key "globally" or within a particular table depending on the iterator used. + /// + /// The "global" or table-restricted iterator to use. + public static void SeekFirst(this IDbIterator iterator) + { + iterator.Seek(new byte[0]); + } + /// /// Gets all the keys in the relevant table subject to any supplied constraints. /// - /// The iterator that also identifies the table being iterated. + /// The "global" or table-restricted iterator to use. /// Defaults to false. Set to true if values should be ommitted - i.e. set to null. /// Defaults to true. Set to false to return keys in ascending order. /// Can be set optionally to specify the lower bound of keys to return. @@ -69,7 +78,7 @@ public static class IDbIteratorExt { // Seek to the first key if it was provided. if (firstKey == null) - iterator.Seek(new byte[0]); + iterator.SeekFirst(); else { iterator.Seek(firstKey); diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index 74af155ef7..1659680341 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -149,7 +149,7 @@ public void Prev() public bool IsValid() { - return this.iterator.Valid() && (!this.table.HasValue || this.iterator.Value()[0] == this.table); + return this.iterator.Valid() && (!this.table.HasValue || this.iterator.Key()[0] == this.table); } public byte[] Key() diff --git a/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs b/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs index db0e2cb484..0b9fc2c054 100644 --- a/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs +++ b/src/Stratis.Features.SQLiteWalletRepository.Tests/WalletRepositoryTests.cs @@ -2,12 +2,15 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; using NBitcoin; using NBitcoin.Protocol; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; +using Stratis.Bitcoin.Features.BlockStore; using Stratis.Bitcoin.Features.BlockStore.Repositories; using Stratis.Bitcoin.Features.ColdStaking; using Stratis.Bitcoin.Features.Wallet; @@ -56,13 +59,22 @@ public MultiWalletRepositoryTests() : base(false) public class BlockBase { public NodeSettings NodeSettings { get; private set; } - public LevelDbBlockRepository BlockRepo { get; private set; } + public BlockRepository BlockRepo { get; private set; } public ChainIndexer ChainIndexer { get; private set; } internal Metrics Metrics { get; set; } public long TicksReading; + /// + /// Using reflection, retrieves the value of private field with this name on the supplied object. If no field is found, returns null. + /// + private object GetPrivateFieldValue(object obj, string fieldName) + { + FieldInfo field = obj.GetType().GetField(fieldName, BindingFlags.NonPublic | BindingFlags.Instance); + return field?.GetValue(obj); + } + public BlockBase(Network network, string dataDir, int blockLimit = int.MaxValue) { // Set up block store. @@ -70,19 +82,21 @@ public BlockBase(Network network, string dataDir, int blockLimit = int.MaxValue) var serializer = new DBreezeSerializer(network.Consensus.ConsensusFactory); // Build the chain from the block store. - this.BlockRepo = new LevelDbBlockRepository(network, this.NodeSettings.DataFolder, serializer); + this.BlockRepo = new BlockRepository(network, this.NodeSettings.DataFolder, serializer); this.BlockRepo.Initialize(); var prevBlock = new Dictionary(); byte[] hashBytes = uint256.Zero.ToBytes(); - using (var itr = this.BlockRepo.Leveldb.GetEnumerator()) + IDb db = (IDb)GetPrivateFieldValue(this.BlockRepo, "db"); + + using (var itr = db.GetIterator(BlockRepositoryConstants.BlockTableName)) { - while (itr.MoveNext()) + foreach ((byte[] key, byte[] value) in itr.GetAll()) { - uint256 hashPrev = serializer.Deserialize(itr.Current.Value); - var hashThis = new uint256(itr.Current.Key); + uint256 hashPrev = serializer.Deserialize(value); + var hashThis = new uint256(key); prevBlock[hashThis] = hashPrev; if (prevBlock.Count >= blockLimit) break; From a235ef6ee390890a77de8142c6065fa5a7aa67c5 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Wed, 10 Aug 2022 18:54:18 +1000 Subject: [PATCH 26/44] Fix MinerPairStarts test case (#1034) (cherry picked from commit c32deea799d2bef560fe14b0525f1361c451f12c) --- .../NodeInitialisationTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs b/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs index 9354f5a475..ceed5b4545 100644 --- a/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs +++ b/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs @@ -133,7 +133,7 @@ public void MainChainGatewayStarts() [Fact] public void MinerPairStarts() { - CirrusRegTest collateralSidechainNetwork = new CirrusSingleCollateralRegTest(); + CirrusRegTest collateralSidechainNetwork = new CirrusSingleCollateralRegTest(this.mainNetwork); using var sideNodeBuilder = SidechainNodeBuilder.CreateSidechainNodeBuilder(this); using var nodeBuilder = NodeBuilder.Create(this); @@ -230,12 +230,12 @@ private static void VerifyNodeComposition(CoreNode node) internal class CirrusSingleCollateralRegTest : CirrusRegTest { - internal CirrusSingleCollateralRegTest() + internal CirrusSingleCollateralRegTest(Network collateralNetwork) { this.Name = "CirrusSingleCollateralRegTest"; var firstMember = this.ConsensusOptions.GenesisFederationMembers[0] as CollateralFederationMember; firstMember.CollateralAmount = Money.Coins(100m); - firstMember.CollateralMainchainAddress = new Key().ScriptPubKey.GetDestinationAddress(this).ToString(); + firstMember.CollateralMainchainAddress = new Key().ScriptPubKey.GetDestinationAddress(collateralNetwork).ToString(); } } From 213b3f075614eacf9ef2396d12fd24edddc29bec Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 18 Aug 2022 17:40:47 +1000 Subject: [PATCH 27/44] Fix CI errors (#1043) (cherry picked from commit ebcebc939e23eb3ba80dd1e727ef4d45e93d1199) --- .../Repositories/BlockRepository.cs | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs index 0aa1a8ad99..28b6a01dc6 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/Repositories/BlockRepository.cs @@ -297,20 +297,21 @@ public void ReIndex() this.logger.LogInformation(warningMessage.ToString()); using (var batch = this.db.GetWriteBatch()) { - var enumerator = this.db.GetIterator(BlockRepositoryConstants.BlockTableName); - - foreach ((byte[] key, byte[] value) in enumerator.GetAll()) + using (var iterator = this.db.GetIterator(BlockRepositoryConstants.BlockTableName)) { - var block = this.dBreezeSerializer.Deserialize(value); - foreach (Transaction transaction in block.Transactions) + foreach ((byte[] key, byte[] value) in iterator.GetAll()) { - batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - } - - // inform the user about the ongoing operation - if (++rowCount % 1000 == 0) - { - this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); + var block = this.dBreezeSerializer.Deserialize(value); + foreach (Transaction transaction in block.Transactions) + { + batch.Put(BlockRepositoryConstants.TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); + } + + // inform the user about the ongoing operation + if (++rowCount % 1000 == 0) + { + this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); + } } } @@ -323,11 +324,13 @@ public void ReIndex() { using (var batch = this.db.GetWriteBatch()) { - var enumerator = this.db.GetIterator(BlockRepositoryConstants.TransactionTableName); - foreach ((byte[] key, _) in enumerator.GetAll(keysOnly: true)) + using (var iterator = this.db.GetIterator(BlockRepositoryConstants.TransactionTableName)) { - // Clear tx from database. - batch.Delete(BlockRepositoryConstants.TransactionTableName, key); + foreach ((byte[] key, _) in iterator.GetAll(keysOnly: true)) + { + // Clear tx from database. + batch.Delete(BlockRepositoryConstants.TransactionTableName, key); + } } batch.Write(); From bad86c771c07f02bae5acf7679dfc630f8365f4a Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 18 Aug 2022 19:39:05 +1000 Subject: [PATCH 28/44] Combine ProvenBlockHeaderRepository implementations (#1041) * Combine ProvenBlockHeaderRepository implementations * Remove unused code (cherry picked from commit 0545d5e184d898bfb1bb2ce223b8b43d0629f639) --- .../ProvenBlockHeaderRepositoryTests.cs | 64 +++--- .../ProvenBlockHeaderStoreTests.cs | 3 +- .../FullNodeBuilderConsensusExtension.cs | 4 +- ...tory.cs => ProvenBlockHeaderRepository.cs} | 39 ++-- .../RocksDbProvenBlockHeaderRepository.cs | 190 ------------------ .../Database/IDbIteratorExt.cs | 2 +- 6 files changed, 65 insertions(+), 237 deletions(-) rename src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/{LevelDbProvenBlockHeaderRepository.cs => ProvenBlockHeaderRepository.cs} (80%) delete mode 100644 src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/RocksDbProvenBlockHeaderRepository.cs diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs index 4f09f7138e..e99315d3a7 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs @@ -3,11 +3,10 @@ using System.Linq; using System.Threading.Tasks; using FluentAssertions; -using LevelDB; using NBitcoin; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; using Stratis.Bitcoin.Interfaces; -using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Tests.Common; using Stratis.Bitcoin.Tests.Common.Logging; using Stratis.Bitcoin.Utilities; @@ -55,10 +54,12 @@ public async Task PutAsync_WritesProvenBlockHeaderAndSavesBlockHashAsync() await repo.PutAsync(items, blockHashHeightPair); } - using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) + using (var db = new LevelDb()) { - var headerOut = this.dBreezeSerializer.Deserialize(engine.Get(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHashHeightPair.Height))); - var hashHeightPairOut = this.DBreezeSerializer.Deserialize(engine.Get(BlockHashHeightTable, new byte[] { 1 })); + db.Open(folder); + + var headerOut = this.dBreezeSerializer.Deserialize(db.Get(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHashHeightPair.Height))); + var hashHeightPairOut = this.DBreezeSerializer.Deserialize(db.Get(BlockHashHeightTable, new byte[] { 1 })); headerOut.Should().NotBeNull(); headerOut.GetHash().Should().Be(provenBlockHeaderIn.GetHash()); @@ -86,17 +87,18 @@ public async Task PutAsync_Inserts_MultipleProvenBlockHeadersAsync() } // Check the ProvenBlockHeader exists in the database. - using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) + using (IDb db = new LevelDb()) { - var headersOut = new Dictionary(); - var enumerator = engine.GetEnumerator(); - while (enumerator.MoveNext()) - if (enumerator.Current.Key[0] == ProvenBlockHeaderTable) - headersOut.Add(enumerator.Current.Key, enumerator.Current.Value); - - headersOut.Keys.Count.Should().Be(2); - this.dBreezeSerializer.Deserialize(headersOut.First().Value).GetHash().Should().Be(items[0].GetHash()); - this.dBreezeSerializer.Deserialize(headersOut.Last().Value).GetHash().Should().Be(items[1].GetHash()); + db.Open(folder); + + using (IDbIterator iterator = db.GetIterator(ProvenBlockHeaderTable)) + { + var headersOut = iterator.GetAll().ToDictionary(i => i.key, i => i.value); + + headersOut.Keys.Count.Should().Be(2); + this.dBreezeSerializer.Deserialize(headersOut.First().Value).GetHash().Should().Be(items[0].GetHash()); + this.dBreezeSerializer.Deserialize(headersOut.Last().Value).GetHash().Should().Be(items[1].GetHash()); + } } } @@ -109,13 +111,19 @@ public async Task GetAsync_ReadsProvenBlockHeaderAsync() int blockHeight = 1; - using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) + using (IDb db = new LevelDb()) { - engine.Put(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight), this.dBreezeSerializer.Serialize(headerIn)); + db.Open(folder); + + using (var batch = db.GetWriteBatch()) + { + batch.Put(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight), this.dBreezeSerializer.Serialize(headerIn)); + batch.Write(); + } } // Query the repository for the item that was inserted in the above code. - using (LevelDbProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder)) + using (IProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder)) { var headerOut = await repo.GetAsync(blockHeight).ConfigureAwait(false); @@ -129,13 +137,19 @@ public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync() { string folder = CreateTestDir(this); - using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) + using (var engine = new LevelDb()) { - engine.Put(ProvenBlockHeaderTable, BitConverter.GetBytes(1), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock())); - engine.Put(BlockHashHeightTable, new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1))); + engine.Open(folder); + + using (var batch = engine.GetWriteBatch()) + { + batch.Put(ProvenBlockHeaderTable, BitConverter.GetBytes(1), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock())); + batch.Put(BlockHashHeightTable, new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1))); + batch.Write(); + } } - using (LevelDbProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder)) + using (IProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder)) { // Select a different block height. ProvenBlockHeader outHeader = await repo.GetAsync(2).ConfigureAwait(false); @@ -173,9 +187,9 @@ public async Task PutAsync_DisposeOnInitialise_ShouldBeAtLastSavedTipAsync() } } - private LevelDbProvenBlockHeaderRepository SetupRepository(Network network, string folder) + private IProvenBlockHeaderRepository SetupRepository(Network network, string folder) { - var repo = new LevelDbProvenBlockHeaderRepository(network, folder, this.LoggerFactory.Object, this.dBreezeSerializer); + var repo = new ProvenBlockHeaderRepository(network, folder, this.LoggerFactory.Object, this.dBreezeSerializer); Task task = repo.InitializeAsync(); @@ -184,4 +198,4 @@ private LevelDbProvenBlockHeaderRepository SetupRepository(Network network, stri return repo; } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderStoreTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderStoreTests.cs index 8d94fe6107..e2c485ad9d 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderStoreTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderStoreTests.cs @@ -8,6 +8,7 @@ using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Networks; @@ -37,7 +38,7 @@ public ProvenBlockHeaderStoreTests() : base(new StraxTest()) var ibdMock = new Mock(); ibdMock.Setup(s => s.IsInitialBlockDownload()).Returns(false); - this.provenBlockHeaderRepository = new LevelDbProvenBlockHeaderRepository(this.Network, CreateTestDir(this), this.LoggerFactory.Object, dBreezeSerializer); + this.provenBlockHeaderRepository = new ProvenBlockHeaderRepository(this.Network, CreateTestDir(this), this.LoggerFactory.Object, dBreezeSerializer); this.provenBlockHeaderStore = new ProvenBlockHeaderStore(DateTimeProvider.Default, this.LoggerFactory.Object, this.provenBlockHeaderRepository, nodeStats, ibdMock.Object); } diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index 06a751fb35..f9e6302a9d 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -69,10 +69,10 @@ public static IFullNodeBuilder UsePosConsensus(this IFullNodeBuilder fullNodeBui services.AddSingleton(); if (coindbType == DbType.Leveldb) - services.AddSingleton(); + services.AddSingleton>(); if (coindbType == DbType.RocksDb) - services.AddSingleton(); + services.AddSingleton>(); }); }); diff --git a/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/LevelDbProvenBlockHeaderRepository.cs b/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs similarity index 80% rename from src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/LevelDbProvenBlockHeaderRepository.cs rename to src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs index 6792b6e72d..6e88471bad 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/LevelDbProvenBlockHeaderRepository.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs @@ -3,12 +3,11 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Interfaces; -using Stratis.Bitcoin.Persistence; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders @@ -16,7 +15,7 @@ namespace Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders /// /// Persistent implementation of the DBreeze repository. /// - public class LevelDbProvenBlockHeaderRepository : IProvenBlockHeaderRepository + public class ProvenBlockHeaderRepository : IProvenBlockHeaderRepository where T : IDb, new() { /// Path the to the database. private readonly string databaseFolder; @@ -27,7 +26,7 @@ public class LevelDbProvenBlockHeaderRepository : IProvenBlockHeaderRepository private readonly ILogger logger; /// Access to the database. - private DB leveldb; + private IDb db; private readonly object locker; @@ -48,7 +47,7 @@ public class LevelDbProvenBlockHeaderRepository : IProvenBlockHeaderRepository /// folder path to the DBreeze database files. /// Factory to create a logger for this type. /// The serializer to use for objects. - public LevelDbProvenBlockHeaderRepository(Network network, DataFolder folder, ILoggerFactory loggerFactory, + public ProvenBlockHeaderRepository(Network network, DataFolder folder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer) : this(network, folder.ProvenBlockHeaderPath, loggerFactory, dBreezeSerializer) { @@ -61,7 +60,7 @@ public LevelDbProvenBlockHeaderRepository(Network network, DataFolder folder, IL /// folder path to the DBreeze database files. /// Factory to create a logger for this type. /// The serializer to use for objects. - public LevelDbProvenBlockHeaderRepository(Network network, string databaseFolder, ILoggerFactory loggerFactory, + public ProvenBlockHeaderRepository(Network network, string databaseFolder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer) { Guard.NotNull(network, nameof(network)); @@ -84,8 +83,8 @@ public Task InitializeAsync() Task task = Task.Run(() => { // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, this.databaseFolder); + this.db = new T(); + this.db.Open(this.databaseFolder); this.TipHashHeight = this.GetTipHash(); @@ -111,7 +110,7 @@ public Task GetAsync(int blockHeight) lock (this.locker) { - row = this.leveldb.Get(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)); + row = this.db.Get(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)); } if (row != null) @@ -155,7 +154,11 @@ private void SetTip(HashHeightPair newTip) lock (this.locker) { - this.leveldb.Put(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey, this.dBreezeSerializer.Serialize(newTip)); + using (var batch = this.db.GetWriteBatch()) + { + batch.Put(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey, this.dBreezeSerializer.Serialize(newTip)); + batch.Write(); + } } } @@ -165,14 +168,14 @@ private void SetTip(HashHeightPair newTip) /// List of items to save. private void InsertHeaders(SortedDictionary headers) { - using (var batch = new WriteBatch()) + lock (this.locker) { - foreach (KeyValuePair header in headers) - batch.Put(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(header.Key), this.dBreezeSerializer.Serialize(header.Value)); - - lock (this.locker) + using (var batch = this.db.GetWriteBatch()) { - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + foreach (KeyValuePair header in headers) + batch.Put(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(header.Key), this.dBreezeSerializer.Serialize(header.Value)); + + batch.Write(); } } } @@ -188,7 +191,7 @@ private HashHeightPair GetTipHash() byte[] row = null; lock (this.locker) { - row = this.leveldb.Get(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey); + row = this.db.Get(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey); } if (row != null) @@ -200,7 +203,7 @@ private HashHeightPair GetTipHash() /// public void Dispose() { - this.leveldb?.Dispose(); + this.db?.Dispose(); } } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/RocksDbProvenBlockHeaderRepository.cs b/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/RocksDbProvenBlockHeaderRepository.cs deleted file mode 100644 index 52803ffcc6..0000000000 --- a/src/Stratis.Bitcoin.Features.Consensus/ProvenBlockHeaders/RocksDbProvenBlockHeaderRepository.cs +++ /dev/null @@ -1,190 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using NBitcoin; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Interfaces; -using Stratis.Bitcoin.Persistence; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders -{ - /// - /// Persistent implementation of the DBreeze repository. - /// - public sealed class RocksDbProvenBlockHeaderRepository : IProvenBlockHeaderRepository - { - private readonly string dataFolder; - private readonly DBreezeSerializer dBreezeSerializer; - private readonly object locker; - private readonly ILogger logger; - private readonly Network network; - private RocksDb rocksDb; - - /// - public HashHeightPair TipHashHeight { get; private set; } - - /// - /// Initializes a new instance of the object. - /// - /// folder path to the DBreeze database files. - /// The serializer to use for objects. - /// Specification of the network the node runs on - RegTest/TestNet/MainNet. - public RocksDbProvenBlockHeaderRepository( - DataFolder dataFolder, - DBreezeSerializer dBreezeSerializer, - Network network) - : this(dataFolder.ProvenBlockHeaderPath, dBreezeSerializer, network) - { - } - - /// - /// Initializes a new instance of the object. - /// - /// folder path to the DBreeze database files. - /// The serializer to use for objects. - /// Specification of the network the node runs on - RegTest/TestNet/MainNet. - public RocksDbProvenBlockHeaderRepository( - string dataFolder, - DBreezeSerializer dBreezeSerializer, - Network network) - { - this.dBreezeSerializer = dBreezeSerializer; - this.dataFolder = dataFolder; - Directory.CreateDirectory(dataFolder); - - this.locker = new object(); - this.logger = LogManager.GetCurrentClassLogger(); - this.network = network; - } - - /// - public Task InitializeAsync() - { - Task task = Task.Run(() => - { - var dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksDb = RocksDb.Open(dbOptions, this.dataFolder); - - this.TipHashHeight = this.GetTipHash(); - - if (this.TipHashHeight != null) - return; - - var hashHeight = new HashHeightPair(this.network.GetGenesis().GetHash(), 0); - - this.SetTip(hashHeight); - - this.TipHashHeight = hashHeight; - }); - - return task; - } - - /// - public Task GetAsync(int blockHeight) - { - Task task = Task.Run(() => - { - byte[] row = null; - - lock (this.locker) - { - row = this.rocksDb.Get(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)); - } - - if (row != null) - return this.dBreezeSerializer.Deserialize(row); - - return null; - }); - - return task; - } - - /// - public Task PutAsync(SortedDictionary headers, HashHeightPair newTip) - { - Guard.NotNull(headers, nameof(headers)); - Guard.NotNull(newTip, nameof(newTip)); - - Guard.Assert(newTip.Hash == headers.Values.Last().GetHash()); - - Task task = Task.Run(() => - { - this.logger.LogDebug("({0}.Count():{1})", nameof(headers), headers.Count()); - - this.InsertHeaders(headers); - - this.SetTip(newTip); - - this.TipHashHeight = newTip; - }); - - return task; - } - - /// - /// Set's the hash and height tip of the new . - /// - /// Hash height pair of the new block tip. - private void SetTip(HashHeightPair newTip) - { - Guard.NotNull(newTip, nameof(newTip)); - - lock (this.locker) - { - this.rocksDb.Put(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey, this.dBreezeSerializer.Serialize(newTip)); - } - } - - /// - /// Inserts items into to the database. - /// - /// List of items to save. - private void InsertHeaders(SortedDictionary headers) - { - using var batch = new WriteBatch(); - { - foreach (KeyValuePair header in headers) - batch.Put(BlockHeaderRepositoryConstants.ProvenBlockHeaderTable, BitConverter.GetBytes(header.Key), this.dBreezeSerializer.Serialize(header.Value)); - - lock (this.locker) - { - this.rocksDb.Write(batch); - } - } - } - - /// - /// Retrieves the current tip from disk. - /// - /// Hash of blocks current tip. - private HashHeightPair GetTipHash() - { - HashHeightPair tipHash = null; - - byte[] row = null; - lock (this.locker) - { - row = this.rocksDb.Get(BlockHeaderRepositoryConstants.BlockHashHeightTable, BlockHeaderRepositoryConstants.BlockHashHeightKey); - } - - if (row != null) - tipHash = this.dBreezeSerializer.Deserialize(row); - - return tipHash; - } - - /// - public void Dispose() - { - this.rocksDb.Dispose(); - } - } -} diff --git a/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs index d4962ffbe4..cb4adccf81 100644 --- a/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs +++ b/src/Stratis.Bitcoin/Database/IDbIteratorExt.cs @@ -31,7 +31,7 @@ public static void SeekFirst(this IDbIterator iterator) /// Defaults to true. Set to false to omit the key specified in . /// Defaults to true. Set to false to omit the key specified in . /// An enumeration containing all the keys and values according to the specified constraints. - public static IEnumerable<(byte[], byte[])> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true, + public static IEnumerable<(byte[] key, byte[] value)> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true, byte[] firstKey = null, byte[] lastKey = null, bool includeFirstKey = true, bool includeLastKey = true) { bool done = false; From d6995891f2bfcae8d71da268f74ce150c9512e1e Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 22 Aug 2022 17:27:03 +1000 Subject: [PATCH 29/44] Resolve separate DBreeze CoinDb implementation (#1036) * Resolve separate DBreeze CoinDb implementation * Add tables * Changes based on feedback (cherry picked from commit 020228a159b276802efb74b21ebe8b34c808f3fa) --- .../CoinViews/CoinviewTests.cs | 2 +- .../CoinViews/Coindb/Coindb.cs | 4 +- .../CoinViews/Coindb/DBreezeCoindb.cs | 383 +----------------- .../FullNodeBuilderConsensusExtension.cs | 2 +- src/Stratis.Bitcoin/Database/DBreezeDb.cs | 232 +++++++++++ src/Stratis.Bitcoin/Database/IDb.cs | 2 +- src/Stratis.Bitcoin/Database/LevelDb.cs | 2 +- src/Stratis.Bitcoin/Database/RocksDb.cs | 2 +- 8 files changed, 248 insertions(+), 381 deletions(-) create mode 100644 src/Stratis.Bitcoin/Database/DBreezeDb.cs diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 4d71c6b4ff..70604e21c6 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -42,7 +42,7 @@ public CoinviewTests() this.loggerFactory = new ExtendedLoggerFactory(); this.nodeStats = new NodeStats(this.dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object); - this.coindb = new DBreezeCoindb(this.network, this.dataFolder, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory)); + this.coindb = new Coindb(this.network, this.dataFolder, this.dateTimeProvider, this.nodeStats, new DBreezeSerializer(this.network.Consensus.ConsensusFactory)); this.coindb.Initialize(); this.chainIndexer = new ChainIndexer(this.network); diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index d7a371a8c6..87fdf73e57 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -213,7 +213,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB { int insertedEntities = 0; - using (var batch = this.coinDb.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch(coinsTable, rewindTable, blockTable)) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -300,7 +300,7 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = this.coinDb.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch(coinsTable, rewindTable, blockTable)) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs index 0864d972a0..f323ee4821 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/DBreezeCoindb.cs @@ -1,381 +1,16 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using DBreeze; -using DBreeze.DataTypes; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Utilities; +using System.Collections.Generic; +using Stratis.Bitcoin.Database; namespace Stratis.Bitcoin.Features.Consensus.CoinViews { - /// - /// Persistent implementation of coinview using dBreeze database. - /// - public class DBreezeCoindb : ICoindb, IStakedb, IDisposable + public class DBreezeDbWithCoinDbNames : DBreezeDb { - /// Database key under which the block hash of the coin view's current tip is stored. - private static readonly byte[] blockHashKey = new byte[0]; - - /// Instance logger. - private readonly ILogger logger; - - /// Specification of the network the node runs on - regtest/testnet/mainnet. - private readonly Network network; - - /// Hash of the block which is currently the tip of the coinview. - private HashHeightPair blockHash; - - /// Performance counter to measure performance of the database insert and query operations. - private readonly BackendPerformanceCounter performanceCounter; - - private BackendPerformanceSnapshot latestPerformanceSnapShot; - - /// Access to dBreeze database. - private readonly DBreezeEngine dBreeze; - - private readonly DBreezeSerializer dBreezeSerializer; - - public DBreezeCoindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, - ILoggerFactory loggerFactory, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) - : this(network, dataFolder.CoindbPath, dateTimeProvider, loggerFactory, nodeStats, dBreezeSerializer) - { - } - - public DBreezeCoindb(Network network, string folder, IDateTimeProvider dateTimeProvider, - ILoggerFactory loggerFactory, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) - { - Guard.NotNull(network, nameof(network)); - Guard.NotEmpty(folder, nameof(folder)); - - this.dBreezeSerializer = dBreezeSerializer; - - // Create the coinview folder if it does not exist. - Directory.CreateDirectory(folder); - - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - this.dBreeze = new DBreezeEngine(folder); - this.network = network; - this.performanceCounter = new BackendPerformanceCounter(dateTimeProvider); - - if (nodeStats.DisplayBenchStats) - nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 300); - } - - public void Initialize() - { - Block genesis = this.network.GetGenesis(); - - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.ValuesLazyLoadingIsOn = false; - transaction.SynchronizeTables("BlockHash"); - - if (this.GetTipHash(transaction) == null) - { - this.SetBlockHash(transaction, new HashHeightPair(genesis.GetHash(), 0)); - - // Genesis coin is unspendable so do not add the coins. - transaction.Commit(); - } - } - } - - public HashHeightPair GetTipHash() - { - HashHeightPair tipHash; - - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.ValuesLazyLoadingIsOn = false; - tipHash = this.GetTipHash(transaction); - } - - return tipHash; - } - - public FetchCoinsResponse FetchCoins(OutPoint[] utxos) - { - FetchCoinsResponse res = new FetchCoinsResponse(); - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.SynchronizeTables("BlockHash", "Coins"); - transaction.ValuesLazyLoadingIsOn = false; - - using (new StopwatchDisposable(o => this.performanceCounter.AddQueryTime(o))) - { - this.performanceCounter.AddQueriedEntities(utxos.Length); - - foreach (OutPoint outPoint in utxos) - { - Row row = transaction.Select("Coins", outPoint.ToBytes()); - Coins outputs = row.Exists ? this.dBreezeSerializer.Deserialize(row.Value) : null; - - this.logger.LogTrace("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); - - res.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, outputs)); - } - } - } - - return res; - } - - /// - /// Obtains a block header hash of the coinview's current tip. - /// - /// Open dBreeze transaction. - /// Block header hash of the coinview's current tip. - private HashHeightPair GetTipHash(DBreeze.Transactions.Transaction transaction) - { - if (this.blockHash == null) - { - Row row = transaction.Select("BlockHash", blockHashKey); - if (row.Exists) - { - this.blockHash = new HashHeightPair(); - this.blockHash.FromBytes(row.Value); - } - } - - return this.blockHash; - } - - private void SetBlockHash(DBreeze.Transactions.Transaction transaction, HashHeightPair nextBlockHash) - { - this.blockHash = nextBlockHash; - transaction.Insert("BlockHash", blockHashKey, nextBlockHash.ToBytes()); - } - - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) - { - int insertedEntities = 0; - - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.ValuesLazyLoadingIsOn = false; - transaction.SynchronizeTables("BlockHash", "Coins", "Rewind"); - - // Speed can degrade when keys are in random order and, especially, if these keys have high entropy. - // This settings helps with speed, see dBreeze documentations about details. - // We should double check if this settings help in our scenario, or sorting keys and operations is enough. - // Refers to issue #2483. https://github.com/stratisproject/StratisBitcoinFullNode/issues/2483 - transaction.Technical_SetTable_OverwriteIsNotAllowed("Coins"); - - using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) - { - HashHeightPair current = this.GetTipHash(transaction); - if (current != oldBlockHash) - { - this.logger.LogTrace("(-)[BLOCKHASH_MISMATCH]"); - throw new InvalidOperationException("Invalid oldBlockHash"); - } - - this.SetBlockHash(transaction, nextBlockHash); - - // Here we'll add items to be inserted in a second pass. - List toInsert = new List(); - - foreach (var coin in unspentOutputs.OrderBy(utxo => utxo.OutPoint, new OutPointComparer())) - { - if (coin.Coins == null) - { - this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint); - transaction.RemoveKey("Coins", coin.OutPoint.ToBytes()); - } - else - { - // Add the item to another list that will be used in the second pass. - // This is for performance reasons: dBreeze is optimized to run the same kind of operations, sorted. - toInsert.Add(coin); - } - } - - for (int i = 0; i < toInsert.Count; i++) - { - var coin = toInsert[i]; - this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count); - - transaction.Insert("Coins", coin.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(coin.Coins)); - } - - if (rewindDataList != null) - { - foreach (RewindData rewindData in rewindDataList) - { - var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1; - - this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - - transaction.Insert("Rewind", nextRewindIndex, this.dBreezeSerializer.Serialize(rewindData)); - } - } - - insertedEntities += unspentOutputs.Count; - transaction.Commit(); - } - } - - this.performanceCounter.AddInsertedEntities(insertedEntities); - } - - /// - /// Creates new disposable DBreeze transaction. - /// - /// Transaction object. - public DBreeze.Transactions.Transaction CreateTransaction() - { - return this.dBreeze.GetTransaction(); - } - - public RewindData GetRewindData(int height) - { - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.SynchronizeTables("BlockHash", "Coins", "Rewind"); - Row row = transaction.Select("Rewind", height); - return row.Exists ? this.dBreezeSerializer.Deserialize(row.Value) : null; - } - } - - /// - public int GetMinRewindHeight() - { - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - Row row = transaction.SelectForward("Rewind").FirstOrDefault(); - - if (!row.Exists) - { - return -1; - } - - return row.Key; - } - } - - /// - public HashHeightPair Rewind(HashHeightPair target) - { - HashHeightPair res = null; - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.SynchronizeTables("BlockHash", "Coins", "Rewind"); - - transaction.ValuesLazyLoadingIsOn = false; - - HashHeightPair current = this.GetTipHash(transaction); - - Row row = transaction.Select("Rewind", current.Height); - - if (!row.Exists) - { - throw new InvalidOperationException($"No rewind data found for block `{current}`"); - } - - transaction.RemoveKey("Rewind", row.Key); - - var rewindData = this.dBreezeSerializer.Deserialize(row.Value); - - this.SetBlockHash(transaction, rewindData.PreviousBlockHash); - - foreach (OutPoint outPoint in rewindData.OutputsToRemove) - { - this.logger.LogTrace("Outputs of outpoint '{0}' will be removed.", outPoint); - transaction.RemoveKey("Coins", outPoint.ToBytes()); - } - - foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) - { - this.logger.LogTrace("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); - transaction.Insert("Coins", rewindDataOutput.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); - } - - res = rewindData.PreviousBlockHash; - - transaction.Commit(); - } - - return res; - } - - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. - public void PutStake(IEnumerable stakeEntries) - { - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.SynchronizeTables("Stake"); - this.PutStakeInternal(transaction, stakeEntries); - transaction.Commit(); - } - } - - /// - /// Persists unsaved POS blocks information to the database. - /// - /// Open dBreeze transaction. - /// List of POS block information to be examined and persists if unsaved. - private void PutStakeInternal(DBreeze.Transactions.Transaction transaction, IEnumerable stakeEntries) - { - foreach (StakeItem stakeEntry in stakeEntries) - { - if (!stakeEntry.InStore) - { - transaction.Insert("Stake", stakeEntry.BlockId.ToBytes(false), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); - stakeEntry.InStore = true; - } - } - } - - /// - /// Retrieves POS blocks information from the database. - /// - /// List of partially initialized POS block information that is to be fully initialized with the values from the database. - public void GetStake(IEnumerable blocklist) - { - using (DBreeze.Transactions.Transaction transaction = this.CreateTransaction()) - { - transaction.SynchronizeTables("Stake"); - transaction.ValuesLazyLoadingIsOn = false; - - foreach (StakeItem blockStake in blocklist) - { - this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - Row stakeRow = transaction.Select("Stake", blockStake.BlockId.ToBytes(false)); - - if (stakeRow.Exists) - { - blockStake.BlockStake = this.dBreezeSerializer.Deserialize(stakeRow.Value); - blockStake.InStore = true; - } - } - } - } - - private void AddBenchStats(StringBuilder log) - { - log.AppendLine(">> DBreezeCoinView Bench"); - - BackendPerformanceSnapshot snapShot = this.performanceCounter.Snapshot(); - - if (this.latestPerformanceSnapShot == null) - log.AppendLine(snapShot.ToString()); - else - log.AppendLine((snapShot - this.latestPerformanceSnapShot).ToString()); - - this.latestPerformanceSnapShot = snapShot; - } - - /// - public void Dispose() + public DBreezeDbWithCoinDbNames() : base(new Dictionary { + { 1, "Coins" }, + { 2, "BlockHash" }, + { 3, "Rewind" }, + { 4, "Stake" } }) { - this.dBreeze.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index f9e6302a9d..460f1014c9 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -84,7 +84,7 @@ public static void ConfigureCoinDatabaseImplementation(this IServiceCollection s switch (coindbType) { case DbType.Dbreeze: - services.AddSingleton(); + services.AddSingleton>(); break; case DbType.Leveldb: diff --git a/src/Stratis.Bitcoin/Database/DBreezeDb.cs b/src/Stratis.Bitcoin/Database/DBreezeDb.cs new file mode 100644 index 0000000000..febf9e6cd8 --- /dev/null +++ b/src/Stratis.Bitcoin/Database/DBreezeDb.cs @@ -0,0 +1,232 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using DBreeze; +using DBreeze.DataTypes; +using DBreeze.Transactions; + +namespace Stratis.Bitcoin.Database +{ + public class BatchContext : IDisposable + { + public Transaction transaction { get; private set; } + + private bool canDispose; + + public BatchContext(Transaction transaction, bool canDispose) + { + this.transaction = transaction; + this.canDispose = canDispose; + } + + public void Dispose() + { + if (this.canDispose) + this.transaction.Dispose(); + } + } + + /// A minimal DBreeze wrapper that makes it compliant with the interface. + public class DBreezeDb : IDb + { + private Dictionary transactions = new Dictionary(); + + private string dbPath; + + private DBreezeEngine db; + + private Dictionary tableNames; + + public DBreezeDb(Dictionary tableNames) + { + this.tableNames = tableNames; + } + + public IDbIterator GetIterator(byte table) + { + return new DBreezeIterator(this, this.tableNames[table]); + } + + public IDbIterator GetIterator() + { + return new DBreezeIterator(this, "default"); + } + + public string GetTableName(byte table) + { + return this.tableNames[table]; + } + + public void Open(string dbPath) + { + this.dbPath = dbPath; + this.db = new DBreezeEngine(dbPath); + } + + public void Clear() + { + this.db.Dispose(); + System.IO.Directory.Delete(this.dbPath, true); + this.db = new DBreezeEngine(this.dbPath); + } + + public IDbBatch GetWriteBatch(params byte[] tables) => new DBreezeBatch(this, tables); + + private (Transaction transaction, bool canDispose) GetTransaction() + { + int threadId = AppDomain.GetCurrentThreadId(); + + // DBreeze does not allow nested transactions on the same thread. + // Re-use any existing transaction. + if (this.transactions.TryGetValue(threadId, out Transaction currentTransaction)) + { + var disposedField = currentTransaction.GetType().GetField("disposed", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + if (!(bool)disposedField.GetValue(currentTransaction)) + return (currentTransaction, false); + this.transactions.Remove(threadId); + } + + var transaction = this.db.GetTransaction(eTransactionTablesLockTypes.EXCLUSIVE); + + this.transactions[threadId] = transaction; + + return (transaction, true); + } + + public BatchContext GetBatchContext(params byte[] tables) + { + (Transaction transaction, bool canDispose) = this.GetTransaction(); + + if (tables.Length != 0) + transaction.SynchronizeTables(tables.Select(t => this.tableNames[t]).ToArray()); + + return new BatchContext(transaction, canDispose); + } + + public byte[] Get(byte table, byte[] key) + { + using (BatchContext ctx = this.GetBatchContext()) + { + return ctx.transaction.Select(this.tableNames[table], key)?.Value; + } + } + + public byte[] Get(byte[] key) + { + using (BatchContext ctx = this.GetBatchContext()) + { + return ctx.transaction.Select("default", key)?.Value; + } + } + + public void Dispose() + { + this.db.Dispose(); + } + } + + /// A minimal LevelDb wrapper that makes it compliant with the interface. + public class DBreezeBatch : IDbBatch + { + private DBreezeDb db; + private BatchContext context; + + public DBreezeBatch(DBreezeDb db, params byte[] tables) : base() + { + this.db = db; + this.context = db.GetBatchContext(tables); + } + + // Methods when using tables. + + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + this.context.transaction.Insert(this.db.GetTableName(table), key, value); + return this; + } + + public IDbBatch Delete(byte table, byte[] key) + { + this.context.transaction.RemoveKey(this.db.GetTableName(table), key); + return this; + } + + // Table-less operations. + + public IDbBatch Put(byte[] key, byte[] value) + { + this.context.transaction.Insert("default", key, value); + return this; + } + + public IDbBatch Delete(byte[] key) + { + this.context.transaction.RemoveKey("default", key); + return this; + } + + public void Write() + { + this.context.transaction.Commit(); + } + + public void Dispose() + { + this.context.Dispose(); + } + } + + /// A minimal LevelDb wrapper that makes it compliant with the interface. + public class DBreezeIterator : IDbIterator + { + private BatchContext context; + private string tableName; + private Row current; + + internal DBreezeIterator(DBreezeDb db, string tableName) + { + this.context = db.GetBatchContext(); + this.tableName = tableName; + } + + public void Seek(byte[] key) + { + this.current = this.context.transaction.SelectForwardStartFrom(this.tableName, key, includeStartFromKey: true, AsReadVisibilityScope: true).FirstOrDefault(); + } + + public void SeekToLast() + { + this.current = this.context.transaction.SelectBackward(this.tableName, AsReadVisibilityScope: true).FirstOrDefault(); + } + + public void Next() + { + this.current = this.context.transaction.SelectForwardStartFrom(this.tableName, this.current.Key, includeStartFromKey: false, AsReadVisibilityScope: true).FirstOrDefault(); + } + + public void Prev() + { + this.current = this.context.transaction.SelectBackwardStartFrom(this.tableName, this.current.Key, includeStartFromKey: false, AsReadVisibilityScope: true).FirstOrDefault(); + } + + public bool IsValid() + { + return this.current?.Exists ?? false; + } + + public byte[] Key() + { + return this.current.Key; + } + + public byte[] Value() + { + return this.current.Value; + } + + public void Dispose() + { + this.context.Dispose(); + } + } +} diff --git a/src/Stratis.Bitcoin/Database/IDb.cs b/src/Stratis.Bitcoin/Database/IDb.cs index 4dea6b6023..efada9197d 100644 --- a/src/Stratis.Bitcoin/Database/IDb.cs +++ b/src/Stratis.Bitcoin/Database/IDb.cs @@ -51,7 +51,7 @@ public interface IDb : IDisposable /// The method will not reflect these changes until they are committed. Use /// the class if uncommitted changes need to be accessed. /// See . - IDbBatch GetWriteBatch(); + IDbBatch GetWriteBatch(params byte[] tables); /// /// Removes all tables and their contents. diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs index 408ac2ebce..066e9b7217 100644 --- a/src/Stratis.Bitcoin/Database/LevelDb.cs +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -33,7 +33,7 @@ public void Clear() this.db = new DB(new Options() { CreateIfMissing = true }, this.dbPath); } - public IDbBatch GetWriteBatch() => new LevelDbBatch(this.db); + public IDbBatch GetWriteBatch(params byte[] tables) => new LevelDbBatch(this.db); public byte[] Get(byte table, byte[] key) { diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs index 1659680341..b17d319d78 100644 --- a/src/Stratis.Bitcoin/Database/RocksDb.cs +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -34,7 +34,7 @@ public void Clear() this.db = RocksDbSharp.RocksDb.Open(new DbOptions().SetCreateIfMissing(), this.dbPath); } - public IDbBatch GetWriteBatch() => new RocksDbBatch(this.db); + public IDbBatch GetWriteBatch(params byte[] tables) => new RocksDbBatch(this.db); public byte[] Get(byte table, byte[] key) { From c4e7433f6a7b793a43b2e5872fdf2735695999ce Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 31 Jan 2023 16:54:18 +1100 Subject: [PATCH 30/44] Fix tests --- src/NBitcoin/ConsensusOptions.cs | 4 ++-- .../PoAConsensusOptions.cs | 16 ++++++++-------- .../API/ApiSteps.cs | 2 +- .../NodeInitialisationTests.cs | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/NBitcoin/ConsensusOptions.cs b/src/NBitcoin/ConsensusOptions.cs index 3afcc6a16d..02cf100913 100644 --- a/src/NBitcoin/ConsensusOptions.cs +++ b/src/NBitcoin/ConsensusOptions.cs @@ -45,10 +45,10 @@ public class ConsensusOptions /// Block Height at which the node should enforce the use of . /// Can be set to zero to indicate that the minimum supported protocol version will not change depending on the block height. - public int EnforceMinProtocolVersionAtBlockHeight { get; protected set; } + public int EnforceMinProtocolVersionAtBlockHeight { get; set; } /// The minimum protocol version which should be used from block height defined in - public ProtocolVersion? EnforcedMinProtocolVersion { get; protected set; } + public ProtocolVersion? EnforcedMinProtocolVersion { get; set; } /// /// Initializes the default values. Currently only used for initialising Bitcoin networks and testing. diff --git a/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs b/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs index 36baf52ef6..3482fe9652 100644 --- a/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs +++ b/src/Stratis.Bitcoin.Features.PoA/PoAConsensusOptions.cs @@ -34,7 +34,7 @@ public class PoAConsensusOptions : ConsensusOptions /// /// This currently only applies to Cirrus Main Net. /// - public uint? FederationMemberActivationTime { get; protected set; } + public uint? FederationMemberActivationTime { get; set; } /// /// The height at which a federation members will be resolved via the class. @@ -47,12 +47,12 @@ public class PoAConsensusOptions : ConsensusOptions /// method which resolves the pubkey from the signature directly. /// /// - public int VotingManagerV2ActivationHeight { get; protected set; } + public int VotingManagerV2ActivationHeight { get; set; } /// /// This is the height on the main chain at which the dynamic fees paid to the multsig for interop conversion requests will activate. /// - public int InterFluxV2MainChainActivationHeight { get; protected set; } + public int InterFluxV2MainChainActivationHeight { get; set; } /// /// The height at which Release 1.3.0.0 became BIP activated. @@ -72,30 +72,30 @@ public class PoAConsensusOptions : ConsensusOptions /// Legacy mining slots are determined by mining_slot = block_height % number_of_federation_members. /// Once the specified height is reached there should no longer be a shift in mining slots when new federation members are added/removed. /// - public int GetMiningTimestampV2ActivationHeight { get; protected set; } + public int GetMiningTimestampV2ActivationHeight { get; set; } /// /// The height at which inituitive mining slots are enfored without any lenience. /// Currently errors are sometimes suppressed if a federation change occurred. /// - public int GetMiningTimestampV2ActivationStrictHeight { get; protected set; } + public int GetMiningTimestampV2ActivationStrictHeight { get; set; } /// /// Logic related to release 1.1.0.0 will activate at this height, this includes Poll Expiry and the Join Federation Voting Request consensus rule. /// - public int Release1100ActivationHeight { get; protected set; } + public int Release1100ActivationHeight { get; set; } /// /// Polls are expired once the tip reaches a block this far beyond the poll start block. /// I.e. if (Math.Max(startblock + PollExpiryBlocks, PollExpiryActivationHeight) <= tip) (See IsPollExpiredAt) /// - public int PollExpiryBlocks { get; protected set; } + public int PollExpiryBlocks { get; set; } /// /// Defines when V2 of the contract serializer will be used. /// I.e if tip <= ContractSerializerV2ActivationHeight, V1 will be used. /// - public int ContractSerializerV2ActivationHeight { get; protected set; } + public int ContractSerializerV2ActivationHeight { get; set; } /// Initializes values for networks that use block size rules. /// See . diff --git a/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs b/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs index 880e71a1d0..56525e4f5c 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/API/ApiSteps.cs @@ -421,7 +421,7 @@ private void a_full_list_of_available_commands_is_returned() { var commands = JsonDataSerializer.Instance.Deserialize>(this.responseText); - commands.Count.Should().Be(39); + commands.Count.Should().Be(41); } private void status_information_is_returned() diff --git a/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs b/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs index ceed5b4545..4d68cefb61 100644 --- a/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs +++ b/src/Stratis.Features.FederatedPeg.IntegrationTests/NodeInitialisationTests.cs @@ -167,7 +167,7 @@ public void GatewayPairStarts() { using (var nodeBuilder = SidechainNodeBuilder.CreateSidechainNodeBuilder(this)) { - CirrusRegTest collateralSidechainNetwork = new CirrusSingleCollateralRegTest(); + CirrusRegTest collateralSidechainNetwork = new CirrusSingleCollateralRegTest(this.mainNetwork); CoreNode side = nodeBuilder.CreateSidechainFederationNode(collateralSidechainNetwork, this.mainNetwork, collateralSidechainNetwork.FederationKeys[0]); side.AppendToConfig("sidechain=1"); From 52a6fe4d887260e56d7136b1f133608e0cf3cd09 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Wed, 5 Oct 2022 13:32:26 +1100 Subject: [PATCH 31/44] Fix attempt to use NFTTransferIndexer with Strax (#1068) --- src/Stratis.CirrusD/Program.cs | 2 +- .../Controllers/Unity3dController.cs | 2 +- src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs | 10 ++++++---- src/Stratis.StraxD/Program.cs | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Stratis.CirrusD/Program.cs b/src/Stratis.CirrusD/Program.cs index 6639fd8989..8d477884b4 100644 --- a/src/Stratis.CirrusD/Program.cs +++ b/src/Stratis.CirrusD/Program.cs @@ -79,7 +79,7 @@ private static IFullNode GetSideChainFullNode(NodeSettings nodeSettings) .UseSmartContractWallet() .AddSQLiteWalletRepository() .UseApi() - .UseUnity3dApi() + .UseUnity3dApi(true) .AddRPC() .AddSignalR(options => { diff --git a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs index e7aa73e7c5..064fd74ec7 100644 --- a/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs +++ b/src/Stratis.Features.Unity3dApi/Controllers/Unity3dController.cs @@ -74,7 +74,7 @@ public class Unity3dController : Controller private readonly INFTTransferIndexer NFTTransferIndexer; public Unity3dController(ILoggerFactory loggerFactory, IAddressIndexer addressIndexer, - IBlockStore blockStore, IChainState chainState, Network network, ICoinView coinView, WalletController walletController, ChainIndexer chainIndexer, INFTTransferIndexer NFTTransferIndexer, + IBlockStore blockStore, IChainState chainState, Network network, ICoinView coinView, WalletController walletController, ChainIndexer chainIndexer, INFTTransferIndexer NFTTransferIndexer = null, IStakeChain stakeChain = null, IContractPrimitiveSerializer primitiveSerializer = null, IStateRepositoryRoot stateRoot = null, IContractAssemblyCache contractAssemblyCache = null, IReceiptRepository receiptRepository = null, ISmartContractTransactionService smartContractTransactionService = null, ILocalExecutor localExecutor = null) diff --git a/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs b/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs index 10c436e6d2..f4d48b8712 100644 --- a/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs +++ b/src/Stratis.Features.Unity3dApi/Unity3dApiFeature.cs @@ -43,7 +43,7 @@ public Unity3dApiFeature( Unity3dApiSettings apiSettings, ILoggerFactory loggerFactory, ICertificateStore certificateStore, - INFTTransferIndexer NFTTransferIndexer) + INFTTransferIndexer NFTTransferIndexer = null /* Only available for Cirrus */) { this.fullNodeBuilder = fullNodeBuilder; this.fullNode = fullNode; @@ -128,7 +128,7 @@ public override void Dispose() this.webHost = null; } - this.NFTTransferIndexer.Dispose(); + this.NFTTransferIndexer?.Dispose(); } } @@ -137,7 +137,7 @@ public override void Dispose() /// public static class Unity3dApiFeatureExtension { - public static IFullNodeBuilder UseUnity3dApi(this IFullNodeBuilder fullNodeBuilder) + public static IFullNodeBuilder UseUnity3dApi(this IFullNodeBuilder fullNodeBuilder, bool isCirrus) { fullNodeBuilder.ConfigureFeature(features => { @@ -148,7 +148,9 @@ public static IFullNodeBuilder UseUnity3dApi(this IFullNodeBuilder fullNodeBuild services.AddSingleton(fullNodeBuilder); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + + if (isCirrus) + services.AddSingleton(); // Controller services.AddTransient(); diff --git a/src/Stratis.StraxD/Program.cs b/src/Stratis.StraxD/Program.cs index 9a030900ab..2371b470cc 100644 --- a/src/Stratis.StraxD/Program.cs +++ b/src/Stratis.StraxD/Program.cs @@ -47,6 +47,7 @@ public static async Task Main(string[] args) .AddSQLiteWalletRepository() .AddPowPosMining(true) .UseApi() + .UseUnity3dApi(false) .AddRPC() .AddSignalR(options => { From 709de1f37215b7fc324391b9ad74834bb5cfefd3 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Fri, 7 Oct 2022 17:14:53 +1100 Subject: [PATCH 32/44] Remove unused checkpoints from CachedCoinView constructor (#1071) (cherry picked from commit a0f638cff513bba44df4c8d7b395769dca526022) --- .../CoinViews/CoinviewTests.cs | 2 +- .../TestChainFactory.cs | 2 +- .../CoinViews/CachedCoinView.cs | 4 +--- src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs | 4 ++-- .../PoW/SmartContractMinerTests.cs | 3 +-- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 70604e21c6..67d1fb1011 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -51,7 +51,7 @@ public CoinviewTests() this.rewindDataIndexCache = new RewindDataIndexCache(this.dateTimeProvider, this.network, new FinalizedBlockInfoRepository(new HashHeightPair()), new Checkpoints()); - this.cachedCoinView = new CachedCoinView(this.network, new Checkpoints(), this.coindb, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new ConsensusSettings(new NodeSettings(this.network)), this.stakeChainStore, this.rewindDataIndexCache); + this.cachedCoinView = new CachedCoinView(this.network, this.coindb, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new ConsensusSettings(new NodeSettings(this.network)), this.stakeChainStore, this.rewindDataIndexCache); this.rewindDataIndexCache.Initialize(this.chainIndexer.Height, this.cachedCoinView); diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index b0a2e528e1..193416a7bc 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -124,7 +124,7 @@ public static async Task CreateAsync(Network network, string d testChainContext.InitialBlockDownloadState = new InitialBlockDownloadState(testChainContext.ChainState, testChainContext.Network, consensusSettings, new Checkpoints(), testChainContext.DateTimeProvider); var inMemoryCoinView = new InMemoryCoinView(new HashHeightPair(testChainContext.ChainIndexer.Tip)); - var cachedCoinView = new CachedCoinView(network, new Checkpoints(), inMemoryCoinView, DateTimeProvider.Default, testChainContext.LoggerFactory, new NodeStats(testChainContext.DateTimeProvider, testChainContext.NodeSettings, new Mock().Object), new ConsensusSettings(testChainContext.NodeSettings)); + var cachedCoinView = new CachedCoinView(network, inMemoryCoinView, DateTimeProvider.Default, testChainContext.LoggerFactory, new NodeStats(testChainContext.DateTimeProvider, testChainContext.NodeSettings, new Mock().Object), new ConsensusSettings(testChainContext.NodeSettings)); var dataFolder = new DataFolder(TestBase.AssureEmptyDir(dataDir).FullName); testChainContext.PeerAddressManager = diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index 86b5279c00..10b9448808 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -127,7 +127,6 @@ public long GetScriptSize private long rewindDataSizeBytes; private DateTime lastCacheFlushTime; private readonly Network network; - private readonly ICheckpoints checkpoints; private readonly IDateTimeProvider dateTimeProvider; private readonly IBlockStore blockStore; private readonly CancellationTokenSource cancellationToken; @@ -136,7 +135,7 @@ public long GetScriptSize private readonly Random random; - public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, + public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); @@ -144,7 +143,6 @@ public CachedCoinView(Network network, ICheckpoints checkpoints, ICoindb coindb, this.coindb = coindb; this.logger = loggerFactory.CreateLogger(this.GetType().FullName); this.network = network; - this.checkpoints = checkpoints; this.dateTimeProvider = dateTimeProvider; this.consensusSettings = consensusSettings; this.stakeChainStore = stakeChainStore; diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index 283d30207e..e5198c749c 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -73,7 +73,7 @@ public void TestCacheCoinView() ChainedHeader chained = this.MakeNext(genesisChainedHeader, ctx.Network); var dateTimeProvider = new DateTimeProvider(); - var cacheCoinView = new CachedCoinView(this.network, new Checkpoints(), ctx.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); + var cacheCoinView = new CachedCoinView(this.network, ctx.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); cacheCoinView.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained)); Assert.NotNull(cacheCoinView.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); @@ -105,7 +105,7 @@ public void CanRewind() using (NodeContext nodeContext = NodeContext.Create(this)) { var dateTimeProvider = new DateTimeProvider(); - var cacheCoinView = new CachedCoinView(this.network, new Checkpoints(), nodeContext.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); + var cacheCoinView = new CachedCoinView(this.network, nodeContext.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); var tester = new CoinViewTester(cacheCoinView); List<(Coins, OutPoint)> coinsA = tester.CreateCoins(5); diff --git a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs index 7794cce1b3..09461b2a40 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs @@ -207,8 +207,7 @@ public async Task InitializeAsync([CallerMemberName] string callingMethod = "") this.NodeSettings = new NodeSettings(this.network, args: new string[] { "-checkpoints" }); var consensusSettings = new ConsensusSettings(this.NodeSettings); - var checkPoints = new Checkpoints(this.network, consensusSettings); - this.cachedCoinView = new CachedCoinView(this.network, checkPoints, inMemoryCoinView, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, this.NodeSettings, new Mock().Object), consensusSettings); + this.cachedCoinView = new CachedCoinView(this.network, inMemoryCoinView, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, this.NodeSettings, new Mock().Object), consensusSettings); var nodeDeployments = new NodeDeployments(this.network, this.ChainIndexer); From 1ef2dd5f3e9ef73c25fbdced3356fa74bec53e71 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 11 Oct 2022 21:44:10 +1100 Subject: [PATCH 33/44] Pass ChainIndexer argument via CachedCoinView constructor (#1072) * Pass ChainIndexer argument via CachedCoinView constructor * Add this * Add missing locks * Refactor (cherry picked from commit cdcc14c54094277392f8a910b509c04db116bd0f) --- .../CoinViews/CoinviewTests.cs | 2 +- .../TestChainFactory.cs | 2 +- .../CoinViews/CachedCoinView.cs | 123 ++++++++++-------- .../CoinViews/CoinView.cs | 8 +- .../CoinViews/InMemoryCoinView.cs | 4 +- .../Rules/CommonRules/LoadCoinviewRule.cs | 2 +- .../Rules/PosConsensusRuleEngine.cs | 4 +- .../Rules/PowConsensusRuleEngine.cs | 8 +- .../MemPoolCoinView.cs | 4 +- .../CoinViewTests.cs | 6 +- .../Consensus/TestInMemoryCoinView.cs | 4 +- src/Stratis.Bitcoin/Base/BaseFeature.cs | 2 +- .../Consensus/ConsensusRuleEngine.cs | 2 +- .../Consensus/IConsensusRuleEngine.cs | 2 +- .../PoW/SmartContractMinerTests.cs | 2 +- 15 files changed, 94 insertions(+), 81 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs index 67d1fb1011..02a7e88324 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/CoinViews/CoinviewTests.cs @@ -51,7 +51,7 @@ public CoinviewTests() this.rewindDataIndexCache = new RewindDataIndexCache(this.dateTimeProvider, this.network, new FinalizedBlockInfoRepository(new HashHeightPair()), new Checkpoints()); - this.cachedCoinView = new CachedCoinView(this.network, this.coindb, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new ConsensusSettings(new NodeSettings(this.network)), this.stakeChainStore, this.rewindDataIndexCache); + this.cachedCoinView = new CachedCoinView(this.network, this.coindb, this.dateTimeProvider, this.loggerFactory, this.nodeStats, new ConsensusSettings(new NodeSettings(this.network)), this.chainIndexer, this.stakeChainStore, this.rewindDataIndexCache); this.rewindDataIndexCache.Initialize(this.chainIndexer.Height, this.cachedCoinView); diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index 193416a7bc..b0aa202729 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -124,7 +124,7 @@ public static async Task CreateAsync(Network network, string d testChainContext.InitialBlockDownloadState = new InitialBlockDownloadState(testChainContext.ChainState, testChainContext.Network, consensusSettings, new Checkpoints(), testChainContext.DateTimeProvider); var inMemoryCoinView = new InMemoryCoinView(new HashHeightPair(testChainContext.ChainIndexer.Tip)); - var cachedCoinView = new CachedCoinView(network, inMemoryCoinView, DateTimeProvider.Default, testChainContext.LoggerFactory, new NodeStats(testChainContext.DateTimeProvider, testChainContext.NodeSettings, new Mock().Object), new ConsensusSettings(testChainContext.NodeSettings)); + var cachedCoinView = new CachedCoinView(network, inMemoryCoinView, DateTimeProvider.Default, testChainContext.LoggerFactory, new NodeStats(testChainContext.DateTimeProvider, testChainContext.NodeSettings, new Mock().Object), new ConsensusSettings(testChainContext.NodeSettings), testChainContext.ChainIndexer); var dataFolder = new DataFolder(TestBase.AssureEmptyDir(dataDir).FullName); testChainContext.PeerAddressManager = diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index 10b9448808..fcc3459ef6 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -131,11 +131,12 @@ public long GetScriptSize private readonly IBlockStore blockStore; private readonly CancellationTokenSource cancellationToken; private readonly ConsensusSettings consensusSettings; + private readonly ChainIndexer chainIndexer; private CachePerformanceSnapshot latestPerformanceSnapShot; private readonly Random random; - public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, + public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer, StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); @@ -145,6 +146,7 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim this.network = network; this.dateTimeProvider = dateTimeProvider; this.consensusSettings = consensusSettings; + this.chainIndexer = chainIndexer; this.stakeChainStore = stakeChainStore; this.rewindDataIndexCache = rewindDataIndexCache; this.blockStore = blockStore; @@ -166,22 +168,22 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim /// /// Remain on-chain. /// - public void Sync(ChainIndexer chainIndexer) + public void Sync() { lock (this.lockobj) { HashHeightPair coinViewTip = this.GetTipHash(); - if (coinViewTip.Hash == chainIndexer.Tip.HashBlock) + if (coinViewTip.Hash == this.chainIndexer.Tip.HashBlock) return; Flush(); - ChainedHeader fork = chainIndexer[coinViewTip.Hash]; + ChainedHeader fork = this.chainIndexer[coinViewTip.Hash]; if (fork == null) { // Determine the last usable height. - int height = BinarySearch.BinaryFindFirst(h => (h > chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1; - fork = chainIndexer[(height < 0) ? coinViewTip.Height : height]; + int height = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1; + fork = this.chainIndexer[(height < 0) ? coinViewTip.Height : height]; } while (coinViewTip.Height != fork.Height) @@ -196,11 +198,13 @@ public void Sync(ChainIndexer chainIndexer) } } - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) + public void Initialize(IConsensusManager consensusManager) { + ChainedHeader chainTip = this.chainIndexer.Tip; + this.coindb.Initialize(); - Sync(chainIndexer); + Sync(); HashHeightPair coinViewTip = this.coindb.GetTipHash(); @@ -209,12 +213,14 @@ public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConse { try { + IConsensusRuleEngine consensusRuleEngine = consensusManager.ConsensusRules; + var loadCoinViewRule = consensusRuleEngine.GetRule(); var saveCoinViewRule = consensusRuleEngine.GetRule(); var coinViewRule = consensusRuleEngine.GetRule(); var deploymentsRule = consensusRuleEngine.GetRule(); - foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(chainIndexer[coinViewTip.Hash], chainIndexer, this.cancellationToken, batchSize: 1000)) + foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(this.chainIndexer[coinViewTip.Hash], this.chainIndexer, this.cancellationToken, batchSize: 1000)) { if (block == null) break; @@ -258,15 +264,18 @@ public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConse public HashHeightPair GetTipHash() { - if (this.blockHash == null) + lock (this.lockobj) { - HashHeightPair response = this.coindb.GetTipHash(); + if (this.blockHash == null) + { + HashHeightPair response = this.coindb.GetTipHash(); - this.innerBlockHash = response; - this.blockHash = this.innerBlockHash; - } + this.innerBlockHash = response; + this.blockHash = this.innerBlockHash; + } - return this.blockHash; + return this.blockHash; + } } /// @@ -412,44 +421,44 @@ private void TryEvictCacheLocked() /// public void Flush(bool force = true) { - if (!force) + lock (this.lockobj) { - // Check if periodic flush is required. - // Ideally this will flush less frequent and always be behind - // blockstore which is currently set to 17 sec. + if (!force) + { + // Check if periodic flush is required. + // Ideally this will flush less frequent and always be behind + // blockstore which is currently set to 17 sec. - DateTime now = this.dateTimeProvider.GetUtcNow(); - bool flushTimeLimit = (now - this.lastCacheFlushTime).TotalSeconds >= this.CacheFlushTimeIntervalSeconds; + DateTime now = this.dateTimeProvider.GetUtcNow(); + bool flushTimeLimit = (now - this.lastCacheFlushTime).TotalSeconds >= this.CacheFlushTimeIntervalSeconds; - // The size of the cache was reached and most likely TryEvictCacheLocked didn't work - // so the cahces is pulledted with flushable items, then we flush anyway. + // The size of the cache was reached and most likely TryEvictCacheLocked didn't work + // so the cahces is pulledted with flushable items, then we flush anyway. - long totalBytes = this.cacheSizeBytes + this.rewindDataSizeBytes; - bool flushSizeLimit = totalBytes > this.MaxCacheSizeBytes; + long totalBytes = this.cacheSizeBytes + this.rewindDataSizeBytes; + bool flushSizeLimit = totalBytes > this.MaxCacheSizeBytes; - if (!flushTimeLimit && !flushSizeLimit) - { - return; - } + if (!flushTimeLimit && !flushSizeLimit) + { + return; + } - this.logger.LogDebug("Flushing, reasons flushTimeLimit={0} flushSizeLimit={1}.", flushTimeLimit, flushSizeLimit); - } + this.logger.LogDebug("Flushing, reasons flushTimeLimit={0} flushSizeLimit={1}.", flushTimeLimit, flushSizeLimit); + } - // Before flushing the coinview persist the stake store - // the stake store depends on the last block hash - // to be stored after the stake store is persisted. - if (this.stakeChainStore != null) - this.stakeChainStore.Flush(true); + // Before flushing the coinview persist the stake store + // the stake store depends on the last block hash + // to be stored after the stake store is persisted. + if (this.stakeChainStore != null) + this.stakeChainStore.Flush(true); - // Before flushing the coinview persist the rewind data index store as well. - if (this.rewindDataIndexCache != null) - this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, null); + // Before flushing the coinview persist the rewind data index store as well. + if (this.rewindDataIndexCache != null) + this.rewindDataIndexCache.SaveAndEvict(this.blockHash.Height, null); - if (this.innerBlockHash == null) - this.innerBlockHash = this.coindb.GetTipHash(); + if (this.innerBlockHash == null) + this.innerBlockHash = this.coindb.GetTipHash(); - lock (this.lockobj) - { if (this.innerBlockHash == null) { this.logger.LogTrace("(-)[NULL_INNER_TIP]"); @@ -476,10 +485,9 @@ public void Flush(bool force = true) this.cachedRewindData.Clear(); this.rewindDataSizeBytes = 0; this.dirtyCacheCount = 0; - this.innerBlockHash = this.blockHash; + this.innerBlockHash = this.blockHash; + this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow(); } - - this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow(); } /// @@ -668,16 +676,16 @@ public void SaveChanges(IList outputs, HashHeightPair oldBlockHas public HashHeightPair Rewind(HashHeightPair target = null) { - if (this.innerBlockHash == null) + lock (this.lockobj) { - this.innerBlockHash = this.coindb.GetTipHash(); - } + if (this.innerBlockHash == null) + { + this.innerBlockHash = this.coindb.GetTipHash(); + } - // Flush the entire cache before rewinding - this.Flush(true); + // Flush the entire cache before rewinding + this.Flush(true); - lock (this.lockobj) - { HashHeightPair hash = this.coindb.Rewind(target); foreach (KeyValuePair cachedUtxoItem in this.cachedUtxoItems) @@ -707,10 +715,13 @@ public HashHeightPair Rewind(HashHeightPair target = null) /// public RewindData GetRewindData(int height) { - if (this.cachedRewindData.TryGetValue(height, out RewindData existingRewindData)) - return existingRewindData; + lock (this.lockobj) + { + if (this.cachedRewindData.TryGetValue(height, out RewindData existingRewindData)) + return existingRewindData; - return this.coindb.GetRewindData(height); + return this.coindb.GetRewindData(height); + } } [NoTrace] diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs index eca612cdb9..8b77dae5fe 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CoinView.cs @@ -14,10 +14,8 @@ public interface ICoinView /// /// Initializes the coin view. /// - /// The chain tip. - /// The chain indexer. - /// The consensus rule engine. - void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine); + /// The consensus manager. + void Initialize(IConsensusManager consensusManager); /// /// Retrieves the block hash of the current tip of the coinview. @@ -46,7 +44,7 @@ public interface ICoinView /// Brings the coinview back on-chain if a re-org occurred. /// /// The current consensus chain. - void Sync(ChainIndexer chainIndexer); + void Sync(); /// /// Obtains information about unspent outputs. diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs index 7652b469a1..4df310e6ef 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/InMemoryCoinView.cs @@ -34,13 +34,13 @@ public InMemoryCoinView(HashHeightPair tipHash) } /// - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) + public void Initialize(IConsensusManager consensusManager) { throw new NotImplementedException(); } /// - public void Sync(ChainIndexer chainIndexer) + public void Sync() { } diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs index bf4d5c26b1..9c70066530 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs @@ -25,7 +25,7 @@ public override Task RunAsync(RuleContext context) // unless the coinview threshold is reached. this.Logger.LogDebug("Saving coinview changes."); var utxoRuleContext = context as UtxoRuleContext; - this.PowParent.UtxoSet.Sync(this.Parent.ChainIndexer); + this.PowParent.UtxoSet.Sync(); this.PowParent.UtxoSet.SaveChanges(utxoRuleContext.UnspentOutputSet.GetCoins(), new HashHeightPair(oldBlock), new HashHeightPair(nextBlock)); return Task.CompletedTask; diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/PosConsensusRuleEngine.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/PosConsensusRuleEngine.cs index 69b224cc04..5102a05ff0 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/PosConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/PosConsensusRuleEngine.cs @@ -48,9 +48,9 @@ public override RuleContext CreateRuleContext(ValidationContext validationContex } /// - public override void Initialize(ChainedHeader chainTip) + public override void Initialize(ChainedHeader chainTip, IConsensusManager consensusManager) { - base.Initialize(chainTip); + base.Initialize(chainTip, consensusManager); this.StakeChain.Load(); diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs index 92319d73db..b0ee7c8f26 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/PowConsensusRuleEngine.cs @@ -63,11 +63,13 @@ public override Task RewindAsync(HashHeightPair target) } /// - public override void Initialize(ChainedHeader chainTip) + public override void Initialize(ChainedHeader chainTip, IConsensusManager consensusManager) { - base.Initialize(chainTip); + base.Initialize(chainTip, consensusManager); - this.UtxoSet.Initialize(chainTip, this.ChainIndexer, this); + Guard.Assert(chainTip.HashBlock == this.ChainIndexer.Tip.HashBlock); + + this.UtxoSet.Initialize(consensusManager); } public override async Task FullValidationAsync(ChainedHeader header, Block block) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs index 79e782b89e..f4862e5e46 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MemPoolCoinView.cs @@ -50,13 +50,13 @@ public MempoolCoinView(Network network, ICoinView inner, ITxMempool memPool, Sch this.Set = new UnspentOutputSet(); } - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) + public void Initialize(IConsensusManager consensusManager) { throw new NotImplementedException(); } /// - public void Sync(ChainIndexer chainIndexer) + public void Sync() { } diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index e5198c749c..40618f9ea8 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -73,7 +73,8 @@ public void TestCacheCoinView() ChainedHeader chained = this.MakeNext(genesisChainedHeader, ctx.Network); var dateTimeProvider = new DateTimeProvider(); - var cacheCoinView = new CachedCoinView(this.network, ctx.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); + var cacheCoinView = new CachedCoinView(this.network, ctx.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), + new Mock().Object), new ConsensusSettings(new NodeSettings(this.network)), new ChainIndexer(this.network)); cacheCoinView.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained)); Assert.NotNull(cacheCoinView.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); @@ -105,7 +106,8 @@ public void CanRewind() using (NodeContext nodeContext = NodeContext.Create(this)) { var dateTimeProvider = new DateTimeProvider(); - var cacheCoinView = new CachedCoinView(this.network, nodeContext.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), new Mock().Object), new ConsensusSettings(new NodeSettings(this.network))); + var cacheCoinView = new CachedCoinView(this.network, nodeContext.Coindb, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, NodeSettings.Default(this.network), + new Mock().Object), new ConsensusSettings(new NodeSettings(this.network)), new ChainIndexer(this.network)); var tester = new CoinViewTester(cacheCoinView); List<(Coins, OutPoint)> coinsA = tester.CreateCoins(5); diff --git a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs index 4c8f8bcfa6..d8ca898d1d 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/TestInMemoryCoinView.cs @@ -35,13 +35,13 @@ public TestInMemoryCoinView(HashHeightPair tipHash) } /// - public void Initialize(ChainedHeader chainTip, ChainIndexer chainIndexer, IConsensusRuleEngine consensusRuleEngine) + public void Initialize(IConsensusManager consensusManager) { throw new NotImplementedException(); } /// - public void Sync(ChainIndexer chainIndexer) + public void Sync() { } diff --git a/src/Stratis.Bitcoin/Base/BaseFeature.cs b/src/Stratis.Bitcoin/Base/BaseFeature.cs index bf3fb3eed7..6a34af0f00 100644 --- a/src/Stratis.Bitcoin/Base/BaseFeature.cs +++ b/src/Stratis.Bitcoin/Base/BaseFeature.cs @@ -296,7 +296,7 @@ public override async Task InitializeAsync() // node shutdown unexpectedly and the finalized block info needs to be reset. this.finalizedBlockInfoRepository.Initialize(this.chainIndexer.Tip); - this.consensusRules.Initialize(this.chainIndexer.Tip); + this.consensusRules.Initialize(this.chainIndexer.Tip, this.consensusManager); await this.consensusManager.InitializeAsync(this.chainIndexer.Tip).ConfigureAwait(false); diff --git a/src/Stratis.Bitcoin/Consensus/ConsensusRuleEngine.cs b/src/Stratis.Bitcoin/Consensus/ConsensusRuleEngine.cs index 5c4b75f4b5..4a44dfef8b 100644 --- a/src/Stratis.Bitcoin/Consensus/ConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin/Consensus/ConsensusRuleEngine.cs @@ -105,7 +105,7 @@ protected ConsensusRuleEngine( } /// - public virtual void Initialize(ChainedHeader chainTip) + public virtual void Initialize(ChainedHeader chainTip, IConsensusManager consensusManager) { this.SetupRulesEngineParent(); } diff --git a/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs b/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs index e255375906..a8bb3b5d78 100644 --- a/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs @@ -19,7 +19,7 @@ public interface IConsensusRuleEngine : IDisposable /// Initialize the rules engine. /// /// Last common header between chain repository and block store if it's available. - void Initialize(ChainedHeader chainTip); + void Initialize(ChainedHeader chainTip, IConsensusManager consensusManager); /// /// Register a new rule to the engine diff --git a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs index 09461b2a40..1ac1127ba7 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs @@ -207,7 +207,7 @@ public async Task InitializeAsync([CallerMemberName] string callingMethod = "") this.NodeSettings = new NodeSettings(this.network, args: new string[] { "-checkpoints" }); var consensusSettings = new ConsensusSettings(this.NodeSettings); - this.cachedCoinView = new CachedCoinView(this.network, inMemoryCoinView, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, this.NodeSettings, new Mock().Object), consensusSettings); + this.cachedCoinView = new CachedCoinView(this.network, inMemoryCoinView, dateTimeProvider, this.loggerFactory, new NodeStats(dateTimeProvider, this.NodeSettings, new Mock().Object), consensusSettings, this.ChainIndexer); var nodeDeployments = new NodeDeployments(this.network, this.ChainIndexer); From fcc3b4e824e7d510d9bce2fd10adbc675795bf53 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 25 Oct 2022 14:51:57 +1100 Subject: [PATCH 34/44] Bump version to 1.5.0.0 (#1077) --- src/FederationSetup/FederationSetup.csproj | 2 +- src/FodyNlogAdapter/FodyNlogAdapter.csproj | 2 +- src/Stratis.Bitcoin.Cli/Stratis.Bitcoin.Cli.csproj | 2 +- .../Stratis.Bitcoin.Features.Api.csproj | 2 +- .../Stratis.Bitcoin.Features.BlockStore.csproj | 2 +- .../Stratis.Bitcoin.Features.ColdStaking.csproj | 2 +- .../Stratis.Bitcoin.Features.Consensus.csproj | 2 +- .../Stratis.Bitcoin.Features.Dns.csproj | 2 +- .../Stratis.Bitcoin.Features.ExternalApi.csproj | 2 +- .../Stratis.Bitcoin.Features.Interop.csproj | 4 ++-- .../Stratis.Bitcoin.Features.LightWallet.csproj | 2 +- .../Stratis.Bitcoin.Features.MemoryPool.csproj | 2 +- .../Stratis.Bitcoin.Features.Miner.csproj | 2 +- .../Stratis.Bitcoin.Features.Notifications.csproj | 2 +- ...atis.Bitcoin.Features.PoA.IntegrationTests.Common.csproj | 2 +- .../Stratis.Bitcoin.Features.PoA.csproj | 2 +- .../Stratis.Bitcoin.Features.RPC.csproj | 2 +- .../Stratis.Bitcoin.Features.SignalR.csproj | 2 +- .../Stratis.Bitcoin.Features.SmartContracts.csproj | 4 ++-- .../Stratis.Bitcoin.Features.Wallet.csproj | 2 +- .../Stratis.Bitcoin.Features.WatchOnlyWallet.csproj | 2 +- .../Stratis.Bitcoin.IntegrationTests.Common.csproj | 2 +- .../Stratis.Bitcoin.Networks.csproj | 6 +++--- .../Stratis.Bitcoin.Tests.Common.csproj | 2 +- src/Stratis.Bitcoin/Properties/AssemblyInfo.cs | 4 ++-- src/Stratis.Bitcoin/Stratis.Bitcoin.csproj | 2 +- src/Stratis.CirrusD/Stratis.CirrusD.csproj | 2 +- src/Stratis.CirrusDnsD/Stratis.CirrusDnsD.csproj | 2 +- src/Stratis.CirrusMinerD/Stratis.CirrusMinerD.csproj | 2 +- src/Stratis.CirrusPegD/Stratis.CirrusPegD.csproj | 2 +- .../Stratis.Features.Diagnostic.csproj | 2 +- .../Stratis.Features.SQLiteWalletRepository.csproj | 2 +- src/Stratis.Features.Unity3dApi/LocalCallContract.cs | 2 +- .../Stratis.Sidechains.Networks.csproj | 2 +- src/Stratis.StraxD/Stratis.StraxD.csproj | 2 +- src/Stratis.StraxDnsD/Stratis.StraxDnsD.csproj | 2 +- 36 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/FederationSetup/FederationSetup.csproj b/src/FederationSetup/FederationSetup.csproj index 03fb098cd9..b1169db04b 100644 --- a/src/FederationSetup/FederationSetup.csproj +++ b/src/FederationSetup/FederationSetup.csproj @@ -2,8 +2,8 @@ Exe + 1.5.0.0 net6.0 - 1.4.0.7 Stratis Group Ltd. diff --git a/src/FodyNlogAdapter/FodyNlogAdapter.csproj b/src/FodyNlogAdapter/FodyNlogAdapter.csproj index 75ed91c0fb..2737853fc2 100644 --- a/src/FodyNlogAdapter/FodyNlogAdapter.csproj +++ b/src/FodyNlogAdapter/FodyNlogAdapter.csproj @@ -3,7 +3,7 @@ net6.0 FodyNlogAdapter - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. Stratis.Utils.FodyNlogAdapter diff --git a/src/Stratis.Bitcoin.Cli/Stratis.Bitcoin.Cli.csproj b/src/Stratis.Bitcoin.Cli/Stratis.Bitcoin.Cli.csproj index 74eeb55cb1..ed38df712a 100644 --- a/src/Stratis.Bitcoin.Cli/Stratis.Bitcoin.Cli.csproj +++ b/src/Stratis.Bitcoin.Cli/Stratis.Bitcoin.Cli.csproj @@ -3,7 +3,7 @@ Exe net6.0 - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.Api/Stratis.Bitcoin.Features.Api.csproj b/src/Stratis.Bitcoin.Features.Api/Stratis.Bitcoin.Features.Api.csproj index 77526bd585..3f54c8265d 100644 --- a/src/Stratis.Bitcoin.Features.Api/Stratis.Bitcoin.Features.Api.csproj +++ b/src/Stratis.Bitcoin.Features.Api/Stratis.Bitcoin.Features.Api.csproj @@ -6,7 +6,7 @@ Stratis.Bitcoin.Features.Api Library Stratis.Features.Api - 1.4.0.7 + 1.5.0.0 False library diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Stratis.Bitcoin.Features.BlockStore.csproj b/src/Stratis.Bitcoin.Features.BlockStore/Stratis.Bitcoin.Features.BlockStore.csproj index 779fb72276..5dad6d2ae9 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Stratis.Bitcoin.Features.BlockStore.csproj +++ b/src/Stratis.Bitcoin.Features.BlockStore/Stratis.Bitcoin.Features.BlockStore.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.ColdStaking/Stratis.Bitcoin.Features.ColdStaking.csproj b/src/Stratis.Bitcoin.Features.ColdStaking/Stratis.Bitcoin.Features.ColdStaking.csproj index a6e412c8be..512a507b07 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking/Stratis.Bitcoin.Features.ColdStaking.csproj +++ b/src/Stratis.Bitcoin.Features.ColdStaking/Stratis.Bitcoin.Features.ColdStaking.csproj @@ -7,7 +7,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.Consensus/Stratis.Bitcoin.Features.Consensus.csproj b/src/Stratis.Bitcoin.Features.Consensus/Stratis.Bitcoin.Features.Consensus.csproj index 70c5bdd849..a27320fb37 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Stratis.Bitcoin.Features.Consensus.csproj +++ b/src/Stratis.Bitcoin.Features.Consensus/Stratis.Bitcoin.Features.Consensus.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.Dns/Stratis.Bitcoin.Features.Dns.csproj b/src/Stratis.Bitcoin.Features.Dns/Stratis.Bitcoin.Features.Dns.csproj index ee2ea71c3f..ab35c1020d 100644 --- a/src/Stratis.Bitcoin.Features.Dns/Stratis.Bitcoin.Features.Dns.csproj +++ b/src/Stratis.Bitcoin.Features.Dns/Stratis.Bitcoin.Features.Dns.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.ExternalAPI/Stratis.Bitcoin.Features.ExternalApi.csproj b/src/Stratis.Bitcoin.Features.ExternalAPI/Stratis.Bitcoin.Features.ExternalApi.csproj index ccd1636c5f..288969efcb 100644 --- a/src/Stratis.Bitcoin.Features.ExternalAPI/Stratis.Bitcoin.Features.ExternalApi.csproj +++ b/src/Stratis.Bitcoin.Features.ExternalAPI/Stratis.Bitcoin.Features.ExternalApi.csproj @@ -1,8 +1,8 @@  + 1.5.0.0 net6.0 - 1.4.0.7 Stratis Group Ltd. Stratis.Features.ExternalAPI Stratis.Features.ExternalAPI diff --git a/src/Stratis.Bitcoin.Features.Interop/Stratis.Bitcoin.Features.Interop.csproj b/src/Stratis.Bitcoin.Features.Interop/Stratis.Bitcoin.Features.Interop.csproj index feca437ce6..6769da2504 100644 --- a/src/Stratis.Bitcoin.Features.Interop/Stratis.Bitcoin.Features.Interop.csproj +++ b/src/Stratis.Bitcoin.Features.Interop/Stratis.Bitcoin.Features.Interop.csproj @@ -1,8 +1,8 @@  - net6.0 - 1.4.0.7 + net6.0 + 1.5.0.0 Stratis Group Ltd. Stratis.Features.Interop Stratis.Features.Interop diff --git a/src/Stratis.Bitcoin.Features.LightWallet/Stratis.Bitcoin.Features.LightWallet.csproj b/src/Stratis.Bitcoin.Features.LightWallet/Stratis.Bitcoin.Features.LightWallet.csproj index f776bbb6b9..ccd3f2c8ed 100644 --- a/src/Stratis.Bitcoin.Features.LightWallet/Stratis.Bitcoin.Features.LightWallet.csproj +++ b/src/Stratis.Bitcoin.Features.LightWallet/Stratis.Bitcoin.Features.LightWallet.csproj @@ -7,7 +7,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Stratis.Bitcoin.Features.MemoryPool.csproj b/src/Stratis.Bitcoin.Features.MemoryPool/Stratis.Bitcoin.Features.MemoryPool.csproj index 46eb5e90a1..ba0d93485c 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Stratis.Bitcoin.Features.MemoryPool.csproj +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Stratis.Bitcoin.Features.MemoryPool.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False library Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.Miner/Stratis.Bitcoin.Features.Miner.csproj b/src/Stratis.Bitcoin.Features.Miner/Stratis.Bitcoin.Features.Miner.csproj index 0dbe278e8f..edec030377 100644 --- a/src/Stratis.Bitcoin.Features.Miner/Stratis.Bitcoin.Features.Miner.csproj +++ b/src/Stratis.Bitcoin.Features.Miner/Stratis.Bitcoin.Features.Miner.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.Notifications/Stratis.Bitcoin.Features.Notifications.csproj b/src/Stratis.Bitcoin.Features.Notifications/Stratis.Bitcoin.Features.Notifications.csproj index 584ca05d7f..c78cd62739 100644 --- a/src/Stratis.Bitcoin.Features.Notifications/Stratis.Bitcoin.Features.Notifications.csproj +++ b/src/Stratis.Bitcoin.Features.Notifications/Stratis.Bitcoin.Features.Notifications.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common.csproj b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common.csproj index bf6d932493..f8fc0a37b1 100644 --- a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common.csproj +++ b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common/Stratis.Bitcoin.Features.PoA.IntegrationTests.Common.csproj @@ -13,7 +13,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False diff --git a/src/Stratis.Bitcoin.Features.PoA/Stratis.Bitcoin.Features.PoA.csproj b/src/Stratis.Bitcoin.Features.PoA/Stratis.Bitcoin.Features.PoA.csproj index 92e06526fd..4ea231be1e 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Stratis.Bitcoin.Features.PoA.csproj +++ b/src/Stratis.Bitcoin.Features.PoA/Stratis.Bitcoin.Features.PoA.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.RPC/Stratis.Bitcoin.Features.RPC.csproj b/src/Stratis.Bitcoin.Features.RPC/Stratis.Bitcoin.Features.RPC.csproj index 15403b1866..4d6d623ffc 100644 --- a/src/Stratis.Bitcoin.Features.RPC/Stratis.Bitcoin.Features.RPC.csproj +++ b/src/Stratis.Bitcoin.Features.RPC/Stratis.Bitcoin.Features.RPC.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.SignalR/Stratis.Bitcoin.Features.SignalR.csproj b/src/Stratis.Bitcoin.Features.SignalR/Stratis.Bitcoin.Features.SignalR.csproj index 31520d8d71..80f97b233d 100644 --- a/src/Stratis.Bitcoin.Features.SignalR/Stratis.Bitcoin.Features.SignalR.csproj +++ b/src/Stratis.Bitcoin.Features.SignalR/Stratis.Bitcoin.Features.SignalR.csproj @@ -1,7 +1,7 @@  + 1.5.0.0 net6.0 - 1.4.0.7 Stratis.Features.SignalR Stratis.Features.SignalR Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/Stratis.Bitcoin.Features.SmartContracts.csproj b/src/Stratis.Bitcoin.Features.SmartContracts/Stratis.Bitcoin.Features.SmartContracts.csproj index 652e132ba5..4b92114426 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/Stratis.Bitcoin.Features.SmartContracts.csproj +++ b/src/Stratis.Bitcoin.Features.SmartContracts/Stratis.Bitcoin.Features.SmartContracts.csproj @@ -1,8 +1,8 @@  - net6.0 - 1.4.0.7 + net6.0 + 1.5.0.0 Stratis Group Ltd. Stratis.Features.SmartContracts Stratis.Features.SmartContracts diff --git a/src/Stratis.Bitcoin.Features.Wallet/Stratis.Bitcoin.Features.Wallet.csproj b/src/Stratis.Bitcoin.Features.Wallet/Stratis.Bitcoin.Features.Wallet.csproj index 3db8c9b429..699f1bd2b2 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/Stratis.Bitcoin.Features.Wallet.csproj +++ b/src/Stratis.Bitcoin.Features.Wallet/Stratis.Bitcoin.Features.Wallet.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Features.WatchOnlyWallet/Stratis.Bitcoin.Features.WatchOnlyWallet.csproj b/src/Stratis.Bitcoin.Features.WatchOnlyWallet/Stratis.Bitcoin.Features.WatchOnlyWallet.csproj index a7e2f4e2c9..4c4bef039d 100644 --- a/src/Stratis.Bitcoin.Features.WatchOnlyWallet/Stratis.Bitcoin.Features.WatchOnlyWallet.csproj +++ b/src/Stratis.Bitcoin.Features.WatchOnlyWallet/Stratis.Bitcoin.Features.WatchOnlyWallet.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.IntegrationTests.Common/Stratis.Bitcoin.IntegrationTests.Common.csproj b/src/Stratis.Bitcoin.IntegrationTests.Common/Stratis.Bitcoin.IntegrationTests.Common.csproj index 7f8accae71..f1ddb34e04 100644 --- a/src/Stratis.Bitcoin.IntegrationTests.Common/Stratis.Bitcoin.IntegrationTests.Common.csproj +++ b/src/Stratis.Bitcoin.IntegrationTests.Common/Stratis.Bitcoin.IntegrationTests.Common.csproj @@ -13,7 +13,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False diff --git a/src/Stratis.Bitcoin.Networks/Stratis.Bitcoin.Networks.csproj b/src/Stratis.Bitcoin.Networks/Stratis.Bitcoin.Networks.csproj index a8b0be4850..7d775041c7 100644 --- a/src/Stratis.Bitcoin.Networks/Stratis.Bitcoin.Networks.csproj +++ b/src/Stratis.Bitcoin.Networks/Stratis.Bitcoin.Networks.csproj @@ -14,9 +14,9 @@ false false false - 1.4.0.7 - 1.4.0.7 - 1.4.0.7 + 1.5.0.0 + 1.5.0.0 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Bitcoin.Tests.Common/Stratis.Bitcoin.Tests.Common.csproj b/src/Stratis.Bitcoin.Tests.Common/Stratis.Bitcoin.Tests.Common.csproj index 696fd1928c..1c052381dc 100644 --- a/src/Stratis.Bitcoin.Tests.Common/Stratis.Bitcoin.Tests.Common.csproj +++ b/src/Stratis.Bitcoin.Tests.Common/Stratis.Bitcoin.Tests.Common.csproj @@ -13,7 +13,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False diff --git a/src/Stratis.Bitcoin/Properties/AssemblyInfo.cs b/src/Stratis.Bitcoin/Properties/AssemblyInfo.cs index bb3b89b91b..a3fab8e310 100644 --- a/src/Stratis.Bitcoin/Properties/AssemblyInfo.cs +++ b/src/Stratis.Bitcoin/Properties/AssemblyInfo.cs @@ -32,6 +32,6 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.4.0.7")] -[assembly: AssemblyFileVersion("1.4.0.7")] +[assembly: AssemblyVersion("1.5.0.0")] +[assembly: AssemblyFileVersion("1.5.0.0")] [assembly: InternalsVisibleTo("Stratis.Bitcoin.Tests")] \ No newline at end of file diff --git a/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj b/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj index 4488a76c77..30627d44fd 100644 --- a/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj +++ b/src/Stratis.Bitcoin/Stratis.Bitcoin.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0-dev False ..\Stratis.ruleset Stratis Group Ltd. diff --git a/src/Stratis.CirrusD/Stratis.CirrusD.csproj b/src/Stratis.CirrusD/Stratis.CirrusD.csproj index e0243ed729..a5807a961f 100644 --- a/src/Stratis.CirrusD/Stratis.CirrusD.csproj +++ b/src/Stratis.CirrusD/Stratis.CirrusD.csproj @@ -2,8 +2,8 @@ Exe + 1.5.0.0 net6.0 - 1.4.0.7 Stratis Group Ltd. Stratis Group Ltd. diff --git a/src/Stratis.CirrusDnsD/Stratis.CirrusDnsD.csproj b/src/Stratis.CirrusDnsD/Stratis.CirrusDnsD.csproj index 2c146e2aaf..eed09e4ac4 100644 --- a/src/Stratis.CirrusDnsD/Stratis.CirrusDnsD.csproj +++ b/src/Stratis.CirrusDnsD/Stratis.CirrusDnsD.csproj @@ -17,7 +17,7 @@ latest Stratis Group Ltd. - 1.4.0.7 + 1.5.0.0 diff --git a/src/Stratis.CirrusMinerD/Stratis.CirrusMinerD.csproj b/src/Stratis.CirrusMinerD/Stratis.CirrusMinerD.csproj index e471e9924f..6ee9fa1822 100644 --- a/src/Stratis.CirrusMinerD/Stratis.CirrusMinerD.csproj +++ b/src/Stratis.CirrusMinerD/Stratis.CirrusMinerD.csproj @@ -2,8 +2,8 @@ Exe + 1.5.0.0 net6.0 - 1.4.0.7 Stratis Group Ltd. Stratis Group Ltd. diff --git a/src/Stratis.CirrusPegD/Stratis.CirrusPegD.csproj b/src/Stratis.CirrusPegD/Stratis.CirrusPegD.csproj index ebb0ab4052..d0537a9d0e 100644 --- a/src/Stratis.CirrusPegD/Stratis.CirrusPegD.csproj +++ b/src/Stratis.CirrusPegD/Stratis.CirrusPegD.csproj @@ -3,7 +3,7 @@ Exe net6.0 - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. diff --git a/src/Stratis.Features.Diagnostic/Stratis.Features.Diagnostic.csproj b/src/Stratis.Features.Diagnostic/Stratis.Features.Diagnostic.csproj index 717149bc30..8f19379bb2 100644 --- a/src/Stratis.Features.Diagnostic/Stratis.Features.Diagnostic.csproj +++ b/src/Stratis.Features.Diagnostic/Stratis.Features.Diagnostic.csproj @@ -4,7 +4,7 @@ net6.0 ..\None.ruleset true - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. diff --git a/src/Stratis.Features.SQLiteWalletRepository/Stratis.Features.SQLiteWalletRepository.csproj b/src/Stratis.Features.SQLiteWalletRepository/Stratis.Features.SQLiteWalletRepository.csproj index 7681cd466d..69069e8249 100644 --- a/src/Stratis.Features.SQLiteWalletRepository/Stratis.Features.SQLiteWalletRepository.csproj +++ b/src/Stratis.Features.SQLiteWalletRepository/Stratis.Features.SQLiteWalletRepository.csproj @@ -14,7 +14,7 @@ false false false - 1.4.0.7 + 1.5.0.0 False Stratis Group Ltd. diff --git a/src/Stratis.Features.Unity3dApi/LocalCallContract.cs b/src/Stratis.Features.Unity3dApi/LocalCallContract.cs index 7f23b65dea..e82a6d4a4b 100644 --- a/src/Stratis.Features.Unity3dApi/LocalCallContract.cs +++ b/src/Stratis.Features.Unity3dApi/LocalCallContract.cs @@ -13,7 +13,7 @@ namespace Stratis.Features.Unity3dApi { - // TODO: Move this to a more central point once 1.4.0.0 stabilises + // TODO: Move this to a more central point once 1.5.0.0 stabilises public interface ILocalCallContract { LocalExecutionResponse LocalCallSmartContract(LocalCallContractRequest request); diff --git a/src/Stratis.Sidechains.Networks/Stratis.Sidechains.Networks.csproj b/src/Stratis.Sidechains.Networks/Stratis.Sidechains.Networks.csproj index 4dff3276e0..1f7332de85 100644 --- a/src/Stratis.Sidechains.Networks/Stratis.Sidechains.Networks.csproj +++ b/src/Stratis.Sidechains.Networks/Stratis.Sidechains.Networks.csproj @@ -5,7 +5,7 @@ Full ..\None.ruleset - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. Stratis.Sidechains.Networks diff --git a/src/Stratis.StraxD/Stratis.StraxD.csproj b/src/Stratis.StraxD/Stratis.StraxD.csproj index 6d227eae98..9117d08e68 100644 --- a/src/Stratis.StraxD/Stratis.StraxD.csproj +++ b/src/Stratis.StraxD/Stratis.StraxD.csproj @@ -16,7 +16,7 @@ latest - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. diff --git a/src/Stratis.StraxDnsD/Stratis.StraxDnsD.csproj b/src/Stratis.StraxDnsD/Stratis.StraxDnsD.csproj index 1c06e4b7e7..d17bd1b432 100644 --- a/src/Stratis.StraxDnsD/Stratis.StraxDnsD.csproj +++ b/src/Stratis.StraxDnsD/Stratis.StraxDnsD.csproj @@ -16,7 +16,7 @@ latest - 1.4.0.7 + 1.5.0.0 Stratis Group Ltd. From 54409723c168f7e3f94e84054a55dff40e4fcfa7 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 27 Oct 2022 22:51:28 +1100 Subject: [PATCH 35/44] Fix GetBlockData (#1079) (cherry picked from commit 17d45204b824dd9c367fbad5556e6557b7043061) --- src/Stratis.Bitcoin/Consensus/ConsensusManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Stratis.Bitcoin/Consensus/ConsensusManager.cs b/src/Stratis.Bitcoin/Consensus/ConsensusManager.cs index 30d372236d..4bdbebf0c8 100644 --- a/src/Stratis.Bitcoin/Consensus/ConsensusManager.cs +++ b/src/Stratis.Bitcoin/Consensus/ConsensusManager.cs @@ -1299,7 +1299,8 @@ public ChainedHeaderBlock[] GetBlockData(List blockHashes) for (int i = 0; i < blocks.Length; i++) { - ChainedHeaderBlock chainedHeaderBlock = blocks[i]; + // Create the chained header block from the chain indexer if required so that the block can still be picked up from the block store. + ChainedHeaderBlock chainedHeaderBlock = blocks[i] ?? new ChainedHeaderBlock(null, this.chainIndexer[blockHashes[i]]); chainedHeaderBlocks[blockHashes[i]] = chainedHeaderBlock; } } From 417026449f28bec732796d79083fcf8abfcf5dad Mon Sep 17 00:00:00 2001 From: quantumagi Date: Sun, 30 Oct 2022 23:39:23 +1100 Subject: [PATCH 36/44] Fix endianness of delete on rewind table (#1080) * Fix endianness of delete on rewind table * Fix duplicate call * Fix consistency * Update mock (cherry picked from commit 81d3ad74ad6bcaf90d9a5fa5e3e42c40a85c6d6c) --- .../AddressIndexerTests.cs | 71 ++++++++-- .../AddressIndexing/AddressIndexer.cs | 2 +- .../CoinViews/CachedCoinView.cs | 72 ++++++---- .../CoinViews/Coindb/Coindb.cs | 53 ++++--- .../Rules/CommonRules/LoadCoinviewRule.cs | 3 +- .../NodeContext.cs | 4 +- .../Consensus/IConsensusRuleEngine.cs | 1 + .../Database/ReadWriteBatch.cs | 130 ++++++++++++++++++ 8 files changed, 276 insertions(+), 60 deletions(-) create mode 100644 src/Stratis.Bitcoin/Database/ReadWriteBatch.cs diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs index 8000b5427b..8bb9f0a27f 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs @@ -1,15 +1,24 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Runtime.InteropServices; +using System.Threading; using LiteDB; using Microsoft.Extensions.DependencyInjection; using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Consensus.Rules; using Stratis.Bitcoin.Controllers.Models; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; +using Stratis.Bitcoin.Features.BlockStore.Repositories; +using Stratis.Bitcoin.Features.Consensus.CoinViews; +using Stratis.Bitcoin.Features.Consensus.Rules; +using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; +using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Networks; using Stratis.Bitcoin.Primitives; using Stratis.Bitcoin.Tests.Common; @@ -26,22 +35,33 @@ public class AddressIndexerTests private readonly Mock consensusManagerMock; + private readonly ChainIndexer chainIndexer; + private readonly Network network; + private readonly IConsensusRuleEngine consensusRuleEngine; + private readonly ChainedHeader genesisHeader; public AddressIndexerTests() { this.network = new StraxMain(); + this.chainIndexer = new ChainIndexer(this.network); + var nodeSettings = new NodeSettings(this.network, args: new[] { "-addressindex", "-txindex" }); + var mockingServices = new ServiceCollection() .AddSingleton(this.network) - .AddSingleton(new StoreSettings(NodeSettings.Default(this.network)) - { - AddressIndex = true, - TxIndex = true - }) + .AddSingleton(nodeSettings) + .AddSingleton(nodeSettings.LoggerFactory) .AddSingleton(new DataFolder(TestBase.CreateTestDir(this))) - .AddSingleton(new ChainIndexer(this.network)) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(typeof(BlockRepository).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder)))) + .AddSingleton() + .AddSingleton(typeof(Coindb).GetConstructors().First(c => c.GetParameters().Any(p => p.ParameterType == typeof(DataFolder)))) + .AddSingleton() + .AddSingleton(this.chainIndexer) .AddSingleton() .AddSingleton(); @@ -49,7 +69,15 @@ public AddressIndexerTests() this.addressIndexer = mockingContext.GetService(); this.genesisHeader = mockingContext.GetService().GetHeader(0); + + var rulesContainer = mockingContext.GetService(); + rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(LoadCoinviewRule)) as FullValidationConsensusRule); + rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SaveCoinviewRule)) as FullValidationConsensusRule); + rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(StraxCoinviewRule)) as FullValidationConsensusRule); + rulesContainer.FullValidationRules.Add(Activator.CreateInstance(typeof(SetActivationDeploymentsFullValidationRule)) as FullValidationConsensusRule); + this.consensusManagerMock = mockingContext.GetService>(); + this.consensusRuleEngine = mockingContext.GetService(); } [Fact] @@ -64,7 +92,7 @@ public void CanInitializeAndDispose() [Fact] public void CanIndexAddresses() { - List headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network); + List headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network, chainIndexer: this.chainIndexer); this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headers.Last()); Script p2pk1 = this.GetRandomP2PKScript(out string address1); @@ -106,7 +134,7 @@ public void CanIndexAddresses() tx.Inputs.Add(new TxIn(new OutPoint(block5.Transactions.First().GetHash(), 0))); var block10 = new Block() { Transactions = new List() { tx } }; - this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny())).Returns((uint256 hash) => + ChainedHeaderBlock GetChainedHeaderBlock(uint256 hash) { ChainedHeader header = headers.SingleOrDefault(x => x.HashBlock == hash); @@ -123,8 +151,26 @@ public void CanIndexAddresses() } return new ChainedHeaderBlock(new Block(), header); + } + + this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny())).Returns((uint256 hash) => + { + return GetChainedHeaderBlock(hash); }); + this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny>())).Returns((List hashes) => + { + return hashes.Select(h => GetChainedHeaderBlock(h)).ToArray(); + }); + + this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny(), It.IsAny(), It.IsAny())).Returns((ChainedHeader header, int size, CancellationTokenSource token) => + { + return headers.Where(h => h.Height > header.Height).Select(h => GetChainedHeaderBlock(h.HashBlock)).ToArray(); + }); + + this.consensusManagerMock.Setup(x => x.ConsensusRules).Returns(this.consensusRuleEngine); + + this.consensusRuleEngine.Initialize(headers.Last(), this.consensusManagerMock.Object); this.addressIndexer.Initialize(); TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headers.Last()); @@ -137,7 +183,7 @@ public void CanIndexAddresses() // Now trigger rewind to see if indexer can handle reorgs. ChainedHeader forkPoint = headers.Single(x => x.Height == 8); - List headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network); + List headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network, chainIndexer: this.chainIndexer); this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny())).Returns((uint256 hash) => { @@ -146,6 +192,11 @@ public void CanIndexAddresses() return new ChainedHeaderBlock(new Block(), headerFork); }); + this.consensusManagerMock.Setup(x => x.GetBlocksAfterBlock(It.IsAny(), It.IsAny(), It.IsAny())).Returns((ChainedHeader header, int size, CancellationTokenSource token) => + { + return headersFork.Where(h => h.Height > header.Height).Select(h => new ChainedHeaderBlock(new Block(), h)).ToArray(); + }); + this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headersFork.Last()); TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headersFork.Last()); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index 7a5b2e23ed..73ea91f8e4 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -297,7 +297,7 @@ private async Task IndexAddressesContinuouslyAsync() if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader) blockToProcess = prefetchedBlock.Block; else - blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock).Block; + blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock)?.Block; if (blockToProcess == null) { diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs index fcc3459ef6..d61572e039 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/CachedCoinView.cs @@ -5,11 +5,12 @@ using System.Threading; using Microsoft.Extensions.Logging; using NBitcoin; +using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Settings; using Stratis.Bitcoin.Consensus; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; using Stratis.Bitcoin.Features.Consensus.Rules.CommonRules; -using Stratis.Bitcoin.Interfaces; +using Stratis.Bitcoin.Primitives; using Stratis.Bitcoin.Utilities; using Stratis.Bitcoin.Utilities.Extensions; using TracerAttributes; @@ -128,16 +129,16 @@ public long GetScriptSize private DateTime lastCacheFlushTime; private readonly Network network; private readonly IDateTimeProvider dateTimeProvider; - private readonly IBlockStore blockStore; private readonly CancellationTokenSource cancellationToken; + private IConsensusManager consensusManager; private readonly ConsensusSettings consensusSettings; private readonly ChainIndexer chainIndexer; private CachePerformanceSnapshot latestPerformanceSnapShot; private readonly Random random; - public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer, - StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, IBlockStore blockStore = null, INodeLifetime nodeLifetime = null) + public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTimeProvider, ILoggerFactory loggerFactory, INodeStats nodeStats, ConsensusSettings consensusSettings, ChainIndexer chainIndexer, + StakeChainStore stakeChainStore = null, IRewindDataIndexCache rewindDataIndexCache = null, INodeLifetime nodeLifetime = null, NodeSettings nodeSettings = null) { Guard.NotNull(coindb, nameof(CachedCoinView.coindb)); @@ -149,7 +150,6 @@ public CachedCoinView(Network network, ICoindb coindb, IDateTimeProvider dateTim this.chainIndexer = chainIndexer; this.stakeChainStore = stakeChainStore; this.rewindDataIndexCache = rewindDataIndexCache; - this.blockStore = blockStore; this.cancellationToken = (nodeLifetime == null) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(nodeLifetime.ApplicationStopping); this.lockobj = new object(); this.cachedUtxoItems = new Dictionary(); @@ -178,34 +178,34 @@ public void Sync() Flush(); - ChainedHeader fork = this.chainIndexer[coinViewTip.Hash]; - if (fork == null) + if (coinViewTip.Height > this.chainIndexer.Height || this.chainIndexer[coinViewTip.Hash] == null) { - // Determine the last usable height. - int height = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || this.GetRewindData(h).PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock, 0, coinViewTip.Height + 1) - 1; - fork = this.chainIndexer[(height < 0) ? coinViewTip.Height : height]; - } + // The coinview tip is above the chain height or on a fork. + // Determine the first unusable height by finding the first rewind data that is not on the consensus chain. + int unusableHeight = BinarySearch.BinaryFindFirst(h => (h > this.chainIndexer.Height) || (this.GetRewindData(h)?.PreviousBlockHash.Hash != this.chainIndexer[h].Previous.HashBlock), 2, coinViewTip.Height - 1); + ChainedHeader fork = this.chainIndexer[unusableHeight - 2]; - while (coinViewTip.Height != fork.Height) - { - if ((coinViewTip.Height % 100) == 0) - this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork); + while (coinViewTip.Height != fork.Height) + { + if ((coinViewTip.Height % 100) == 0) + this.logger.LogInformation("Rewinding coin view from '{0}' to {1}.", coinViewTip, fork); + + // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. + // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. + coinViewTip = this.coindb.Rewind(new HashHeightPair(fork)); + }; + + this.blockHash = coinViewTip; + this.innerBlockHash = this.blockHash; + } - // If the block store was initialized behind the coin view's tip, rewind it to on or before it's tip. - // The node will complete loading before connecting to peers so the chain will never know that a reorg happened. - coinViewTip = this.coindb.Rewind(new HashHeightPair(fork)); - }; + CatchUp(); } } - public void Initialize(IConsensusManager consensusManager) + private void CatchUp() { ChainedHeader chainTip = this.chainIndexer.Tip; - - this.coindb.Initialize(); - - Sync(); - HashHeightPair coinViewTip = this.coindb.GetTipHash(); // If the coin view is behind the block store then catch up from the block store. @@ -213,15 +213,20 @@ public void Initialize(IConsensusManager consensusManager) { try { - IConsensusRuleEngine consensusRuleEngine = consensusManager.ConsensusRules; + IConsensusRuleEngine consensusRuleEngine = this.consensusManager.ConsensusRules; var loadCoinViewRule = consensusRuleEngine.GetRule(); var saveCoinViewRule = consensusRuleEngine.GetRule(); var coinViewRule = consensusRuleEngine.GetRule(); var deploymentsRule = consensusRuleEngine.GetRule(); - foreach ((ChainedHeader chainedHeader, Block block) in this.blockStore.BatchBlocksFrom(this.chainIndexer[coinViewTip.Hash], this.chainIndexer, this.cancellationToken, batchSize: 1000)) + foreach (ChainedHeaderBlock chb in this.consensusManager.GetBlocksAfterBlock(this.chainIndexer[coinViewTip.Hash], 1000, this.cancellationToken)) { + ChainedHeader chainedHeader = chb?.ChainedHeader; + if (chainedHeader == null) + break; + + Block block = chb.Block; if (block == null) break; @@ -244,7 +249,7 @@ public void Initialize(IConsensusManager consensusManager) coinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); // Saves the changes to the coinview. - saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); + saveCoinViewRule.RunAsync(utxoRuleContext).ConfigureAwait(false).GetAwaiter().GetResult(); } } finally @@ -258,6 +263,15 @@ public void Initialize(IConsensusManager consensusManager) } } } + } + + public void Initialize(IConsensusManager consensusManager) + { + this.consensusManager = consensusManager; + + this.coindb.Initialize(); + + Sync(); this.logger.LogInformation("Coin view initialized at '{0}'.", this.coindb.GetTipHash()); } @@ -485,7 +499,7 @@ public void Flush(bool force = true) this.cachedRewindData.Clear(); this.rewindDataSizeBytes = 0; this.dirtyCacheCount = 0; - this.innerBlockHash = this.blockHash; + this.innerBlockHash = this.blockHash; this.lastCacheFlushTime = this.dateTimeProvider.GetUtcNow(); } } diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index 87fdf73e57..e04c729ee8 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -50,17 +50,11 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) - : this(network, dataFolder.CoindbPath, dateTimeProvider, nodeStats, dBreezeSerializer) - { - } - - public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) { Guard.NotNull(network, nameof(network)); - Guard.NotEmpty(dataFolder, nameof(dataFolder)); + Guard.NotNull(dataFolder, nameof(dataFolder)); - this.dataFolder = dataFolder; + this.dataFolder = dataFolder.CoindbPath; this.dBreezeSerializer = dBreezeSerializer; this.logger = LogManager.GetCurrentClassLogger(); this.network = network; @@ -70,14 +64,13 @@ public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProv nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); } + /// public void Initialize() { // Open a connection to a new DB and create if not found this.coinDb = new T(); this.coinDb.Open(this.dataFolder); - EndiannessFix(); - EnsureCoinDatabaseIntegrity(); Block genesis = this.network.GetGenesis(); @@ -156,6 +149,8 @@ private void EnsureCoinDatabaseIntegrity() { this.logger.LogInformation("Checking coin database integrity..."); + this.EndiannessFix(); + if (this.GetTipHash() == null) { this.logger.LogInformation($"Rebuilding coin database that has no tip information."); @@ -211,9 +206,12 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) { + if (unspentOutputs.Count == 0 && rewindDataList.Count == 0) + return; + int insertedEntities = 0; - using (var batch = this.coinDb.GetWriteBatch(coinsTable, rewindTable, blockTable)) + using (var batch = this.coinDb.GetReadWriteBatch(coinsTable, rewindTable, blockTable)) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -290,6 +288,20 @@ public int GetMinRewindHeight() } } + private bool TryGetCoins(ReadWriteBatch readWriteBatch, byte[] key, out Coins coins) + { + byte[] row2 = readWriteBatch.Get(coinsTable, key); + if (row2 == null) + { + coins = null; + return false; + } + + coins = this.dBreezeSerializer.Deserialize(row2); + + return true; + } + public HashHeightPair Rewind(HashHeightPair target) { HashHeightPair current = this.GetTipHash(); @@ -300,23 +312,32 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = this.coinDb.GetWriteBatch(coinsTable, rewindTable, blockTable)) + using (var batch = this.coinDb.GetReadWriteBatch(coinsTable, rewindTable, blockTable)) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] rowKey = BitConverter.GetBytes(height).Reverse().ToArray(); + byte[] row = this.coinDb.Get(rewindTable, rowKey); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); - batch.Delete(rewindTable, BitConverter.GetBytes(height)); + batch.Delete(rewindTable, rowKey); var rewindData = this.dBreezeSerializer.Deserialize(row); foreach (OutPoint outPoint in rewindData.OutputsToRemove) { - this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(coinsTable, outPoint.ToBytes()); + byte[] key = outPoint.ToBytes(); + if (this.TryGetCoins(batch, key, out Coins coins)) + { + this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); + batch.Delete(coinsTable, key); + } + else + { + throw new InvalidOperationException(string.Format("Outputs of outpoint '{0}' were not found when attempting removal.", outPoint)); + } } foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) diff --git a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs index 9c70066530..3c42d49ee7 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/Rules/CommonRules/LoadCoinviewRule.cs @@ -25,7 +25,6 @@ public override Task RunAsync(RuleContext context) // unless the coinview threshold is reached. this.Logger.LogDebug("Saving coinview changes."); var utxoRuleContext = context as UtxoRuleContext; - this.PowParent.UtxoSet.Sync(); this.PowParent.UtxoSet.SaveChanges(utxoRuleContext.UnspentOutputSet.GetCoins(), new HashHeightPair(oldBlock), new HashHeightPair(nextBlock)); return Task.CompletedTask; @@ -67,7 +66,7 @@ public override Task RunAsync(RuleContext context) { // Check that the current block has not been reorged. // Catching a reorg at this point will not require a rewind. - if (context.ValidationContext.BlockToValidate.Header.HashPrevBlock != this.PowParent.UtxoSet.GetTipHash().Hash) + if (context.ValidationContext.ChainedHeaderToValidate.Previous.HashBlock != this.PowParent.UtxoSet.GetTipHash().Hash) { this.Logger.LogDebug("Reorganization detected."); ConsensusErrors.InvalidPrevTip.Throw(); diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index b503f3ea3d..58f39ef1eb 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -28,7 +28,7 @@ public NodeContext(object caller, string name, Network network) this.FolderName = TestBase.CreateTestDir(caller, name); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); + this.Coindb = new Coindb(network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); this.Coindb.Initialize(); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -66,7 +66,7 @@ public void ReloadPersistentCoinView() this.cleanList.Remove((IDisposable)this.Coindb); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new Coindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); + this.Coindb = new Coindb(this.Network, new DataFolder(this.FolderName), dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); this.Coindb.Initialize(); this.cleanList.Add((IDisposable)this.Coindb); diff --git a/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs b/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs index a8bb3b5d78..2ff5f1167d 100644 --- a/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs +++ b/src/Stratis.Bitcoin/Consensus/IConsensusRuleEngine.cs @@ -19,6 +19,7 @@ public interface IConsensusRuleEngine : IDisposable /// Initialize the rules engine. /// /// Last common header between chain repository and block store if it's available. + /// The consensus manager. void Initialize(ChainedHeader chainTip, IConsensusManager consensusManager); /// diff --git a/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs b/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs new file mode 100644 index 0000000000..6964019b48 --- /dev/null +++ b/src/Stratis.Bitcoin/Database/ReadWriteBatch.cs @@ -0,0 +1,130 @@ +using System.Collections.Generic; +using System.Linq; +using NBitcoin; + +namespace Stratis.Bitcoin.Database +{ + /// + /// A batch that can be used to record changes that can be applied atomically. + /// + /// The supplied method will immediately reflect any changes that have + /// been made or retrieve the value from the underlying database. In contrast the database method + /// will only show the changes after the method is called. + public class ReadWriteBatch : IDbBatch + { + private readonly IDb db; + private readonly IDbBatch batch; + private Dictionary cache; + + public ReadWriteBatch(IDb db, params byte[] tables) + { + this.db = db; + this.batch = db.GetWriteBatch(tables); + this.cache = new Dictionary(new ByteArrayComparer()); + } + + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This class for fluent operations. + public IDbBatch Put(byte[] key, byte[] value) + { + this.cache[key] = value; + return this.batch.Put(key, value); + } + + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This interface for fluent operations. + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = value; + return this.batch.Put(table, key, value); + } + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table key that will be removed. + /// This interface for fluent operations. + public IDbBatch Delete(byte[] key) + { + this.cache[key] = null; + return this.batch.Delete(key); + } + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that will be removed. + /// This interface for fluent operations. + public IDbBatch Delete(byte table, byte[] key) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = null; + return this.batch.Delete(table, key); + } + + /// + /// Returns any changes that have been made to the batch or retrieves the value from the underlying database.. + /// + /// The table key of the value to retrieve. + /// This interface for fluent operations. + public byte[] Get(byte[] key) + { + if (this.cache.TryGetValue(key, out byte[] value)) + return value; + + return this.db.Get(key); + } + + /// + /// Returns any changes that have been made to the batch or retrieves the value from the underlying database.. + /// + /// The table of the value to be retrieved. + /// The table key of the value to retrieve. + /// This interface for fluent operations. + public byte[] Get(byte table, byte[] key) + { + if (this.cache.TryGetValue(new byte[] { table }.Concat(key).ToArray(), out byte[] value)) + return value; + + return this.db.Get(table, key); + } + + /// + /// Writes the recorded changes to the database. + /// + public void Write() + { + this.batch.Write(); + } + + public void Dispose() + { + this.batch.Dispose(); + } + } + + /// + /// Extension methods that build on the interface. + /// + public static class IDbExt + { + /// + /// Gets a . + /// + /// The database to get the batch for. + /// The . + public static ReadWriteBatch GetReadWriteBatch(this IDb db, params byte[] tables) + { + return new ReadWriteBatch(db, tables); + } + } +} \ No newline at end of file From 901ba40da0a499a9a3a28bd579573837796d5d34 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 31 Oct 2022 17:38:00 +1100 Subject: [PATCH 37/44] Fix EnableAutoKickAsync (#1082) * Fix EnableAutoKickAsync * Improve comment formatting (cherry picked from commit a569e0bc159d5a81a8e368247a968af4ea471919) --- .../EnableVoteKickingTests.cs | 6 ++++-- src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests/EnableVoteKickingTests.cs b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests/EnableVoteKickingTests.cs index 97f8074144..9608704970 100644 --- a/src/Stratis.Bitcoin.Features.PoA.IntegrationTests/EnableVoteKickingTests.cs +++ b/src/Stratis.Bitcoin.Features.PoA.IntegrationTests/EnableVoteKickingTests.cs @@ -32,7 +32,8 @@ public async Task EnableAutoKickAsync() targetSpacingSeconds: 60, votingEnabled: true, autoKickIdleMembers: false, - federationMemberMaxIdleTimeSeconds: oldOptions.FederationMemberMaxIdleTimeSeconds); + federationMemberMaxIdleTimeSeconds: oldOptions.FederationMemberMaxIdleTimeSeconds, + pollExpiryBlocks: 450); CoreNode node1 = builder.CreatePoANode(votingNetwork1, votingNetwork1.FederationKey1).Start(); CoreNode node2 = builder.CreatePoANode(votingNetwork2, votingNetwork2.FederationKey2).Start(); @@ -54,7 +55,8 @@ public async Task EnableAutoKickAsync() targetSpacingSeconds: 60, votingEnabled: true, autoKickIdleMembers: true, - federationMemberMaxIdleTimeSeconds: idleTimeSeconds); + federationMemberMaxIdleTimeSeconds: idleTimeSeconds, + pollExpiryBlocks: 450); // Restart node 1 to ensure that we have the new network consensus options which reflects // the autokicking enabled. diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs index 9f99c71afc..7d5e8044b2 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs @@ -87,7 +87,6 @@ public VotingManager( this.locker = new object(); this.votingDataEncoder = new VotingDataEncoder(); this.scheduledVotingData = new List(); - this.PollsRepository = new PollsRepository(chainIndexer, dataFolder, dBreezeSerializer, network as PoANetwork); this.logger = LogManager.GetCurrentClassLogger(); this.network = network; @@ -100,6 +99,10 @@ public VotingManager( this.nodeLifetime = nodeLifetime; this.nodeDeployments = nodeDeployments; + // Avoid hiding the above "Assert" errors by doing this last. + // Otherwise we will just see database file in-use error when this constructor is called on dispose. + this.PollsRepository = new PollsRepository(chainIndexer, dataFolder, dBreezeSerializer, network as PoANetwork); + this.isInitialized = false; } From 7d49e0bd114a8562829030a414b4ee21bbe6ed1e Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 31 Oct 2022 17:40:55 +1100 Subject: [PATCH 38/44] Fix TestDBreezeSerialization test case (#1081) (cherry picked from commit 632cdbdb0b3cfe38297322bae8a2d5cc2107167a) --- .../CoinViews/Coindb/Coindb.cs | 23 ++++++++++--------- .../CoinViews/Coindb/ICoindb.cs | 2 +- .../CoinViewTests.cs | 6 ++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index e04c729ee8..cb703fb8af 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -204,15 +204,19 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) return res; } - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) + public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList) { - if (unspentOutputs.Count == 0 && rewindDataList.Count == 0) - return; - int insertedEntities = 0; using (var batch = this.coinDb.GetReadWriteBatch(coinsTable, rewindTable, blockTable)) { + if (unspentOutputs.Count == 0 && rewindDataList.Count == 0) + { + this.SetBlockHash(batch, nextBlockHash); + batch.Write(); + return; + } + using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { HashHeightPair current = this.GetTipHash(); @@ -248,16 +252,13 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB batch.Put(coinsTable, coin.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(coin.Coins)); } - if (rewindDataList != null) + foreach (RewindData rewindData in rewindDataList) { - foreach (RewindData rewindData in rewindDataList) - { - var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1; + var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1; - this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); + this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - batch.Put(rewindTable, BitConverter.GetBytes(nextRewindIndex).Reverse().ToArray(), this.dBreezeSerializer.Serialize(rewindData)); - } + batch.Put(rewindTable, BitConverter.GetBytes(nextRewindIndex).Reverse().ToArray(), this.dBreezeSerializer.Serialize(rewindData)); } insertedEntities += unspentOutputs.Count; diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index 076f696aed..01cd11ace5 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -33,7 +33,7 @@ public interface ICoindb /// Block hash of the current tip of the coinview. /// Block hash of the tip of the coinview after the change is applied. /// List of rewind data items to be persisted. - void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null); + void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList); /// /// Obtains information about unspent outputs. diff --git a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs index 40618f9ea8..28fade3589 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/CoinViewTests.cs @@ -40,21 +40,21 @@ public CoinViewTests() } [Fact] - public void TestDBreezeSerialization() + public void TestDatabaseSerialization() { using (NodeContext ctx = NodeContext.Create(this)) { Block genesis = ctx.Network.GetGenesis(); var genesisChainedHeader = new ChainedHeader(genesis.Header, ctx.Network.GenesisHash, 0); ChainedHeader chained = this.MakeNext(genesisChainedHeader, ctx.Network); - ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained)); + ctx.Coindb.SaveChanges(new UnspentOutput[] { new UnspentOutput(new OutPoint(genesis.Transactions[0], 0), new Coins(0, genesis.Transactions[0].Outputs.First(), true)) }, new HashHeightPair(genesisChainedHeader), new HashHeightPair(chained), new List()); Assert.NotNull(ctx.Coindb.FetchCoins(new[] { new OutPoint(genesis.Transactions[0], 0) }).UnspentOutputs.Values.FirstOrDefault().Coins); Assert.Null(ctx.Coindb.FetchCoins(new[] { new OutPoint() }).UnspentOutputs.Values.FirstOrDefault().Coins); ChainedHeader previous = chained; chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); chained = this.MakeNext(this.MakeNext(genesisChainedHeader, ctx.Network), ctx.Network); - ctx.Coindb.SaveChanges(new List(), new HashHeightPair(previous), new HashHeightPair(chained)); + ctx.Coindb.SaveChanges(new List(), new HashHeightPair(previous), new HashHeightPair(chained), new List()); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); ctx.ReloadPersistentCoinView(); Assert.Equal(chained.HashBlock, ctx.Coindb.GetTipHash().Hash); From cb3d3333891bd9c00c442968607dbc258f444e73 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 1 Nov 2022 21:49:00 +1100 Subject: [PATCH 39/44] Optimise GetLastBalanceDecreaseTransaction (#1085) * Optimise GeLastBalanceDecreaseTransaction * Fix indentation --- .../AddressIndexing/AddressIndexer.cs | 6 ++++-- src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs | 2 ++ src/Stratis.Bitcoin.Features.BlockStore/IBlockRepository.cs | 3 --- src/Stratis.Bitcoin/Interfaces/IBlockStore.cs | 3 +++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs index 73ea91f8e4..01ec6754bf 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -70,6 +70,7 @@ public class AddressIndexer : IAddressIndexer private readonly IAsyncProvider asyncProvider; private readonly IScriptAddressReader scriptAddressReader; + private readonly IBlockStore blockStore; private readonly TimeSpan flushChangesInterval; @@ -159,6 +160,7 @@ public AddressIndexer(StoreSettings storeSettings, DataFolder dataFolder, Networ this.dateTimeProvider = dateTimeProvider; this.utxoIndexer = utxoIndexer; this.scriptAddressReader = new ScriptAddressReader(); + this.blockStore = blockStore; this.signals = signals; this.lockObject = new object(); @@ -714,7 +716,7 @@ public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(str return null; // Get the UTXO snapshot as of one block lower than the last balance change, so that we are definitely able to look up the inputs of each transaction in the next block. - ReconstructedCoinviewContext utxos = this.utxoIndexer.GetCoinviewAtHeight(lastBalanceHeight - 1); + ReconstructedCoinviewContext utxos = this.blockStore.TxIndex ? null : this.utxoIndexer.GetCoinviewAtHeight(lastBalanceHeight - 1); Transaction foundTransaction = null; @@ -725,7 +727,7 @@ public LastBalanceDecreaseTransactionModel GetLastBalanceDecreaseTransaction(str foreach (TxIn txIn in transaction.Inputs) { - Transaction prevTx = utxos.Transactions[txIn.PrevOut.Hash]; + Transaction prevTx = utxos?.Transactions[txIn.PrevOut.Hash] ?? this.blockStore.GetTransactionById(txIn.PrevOut.Hash); foreach (TxOut txOut in prevTx.Outputs) { diff --git a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs index 60d1ac6285..2a0a93fb9c 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs @@ -100,6 +100,8 @@ public class BlockStoreQueue : IBlockStoreQueue private Exception saveAsyncLoopException; + public bool TxIndex => this.blockRepository.TxIndex; + public BlockStoreQueue( ChainIndexer chainIndexer, IChainState chainState, diff --git a/src/Stratis.Bitcoin.Features.BlockStore/IBlockRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/IBlockRepository.cs index 37c50287dd..65aa94b62a 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/IBlockRepository.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/IBlockRepository.cs @@ -62,8 +62,5 @@ public interface IBlockRepository : IBlockStore /// Hash and height of the repository's tip. HashHeightPair TipHashAndHeight { get; } - - /// Indicates that the node should store all transaction data in the database. - bool TxIndex { get; } } } diff --git a/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs b/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs index c0c142364e..8ce9e5e097 100644 --- a/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs +++ b/src/Stratis.Bitcoin/Interfaces/IBlockStore.cs @@ -37,6 +37,9 @@ public interface IBlockStore : IDisposable Block GetBlock(uint256 blockHash); List GetBlocks(List blockHashes); + + /// Indicates that the node should store all transaction data in the database. + bool TxIndex { get; } } /// From d5706ba7400d44f2f2c6622c585d6d7077641072 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 8 Nov 2022 18:27:32 +1100 Subject: [PATCH 40/44] Refactor WhitelistedHashesRepository (#1090) * Refactor WhitelistedHashesRepository * Add missing bits * Fix constructor * Fix * Enforce poll application in height and id order * Make UnProcessBlock deterministic --- .../VotingManagerTests.cs | 4 +- .../PoAFeature.cs | 2 +- .../Voting/PollResultExecutor.cs | 28 ++-- .../Voting/VotingManager.cs | 17 ++- .../Voting/WhitelistedHashesRepository.cs | 132 ++++++++++++++---- .../Rules/AllowedCodeHashLogicTests.cs | 14 +- .../WhitelistHashCheckerTests.cs | 12 +- .../PoA/Rules/AllowedCodeHashLogic.cs | 2 +- .../PoA/WhitelistedHashChecker.cs | 6 +- .../WhitelistNodeExtensions.cs | 2 +- 10 files changed, 150 insertions(+), 69 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.PoA.Tests/VotingManagerTests.cs b/src/Stratis.Bitcoin.Features.PoA.Tests/VotingManagerTests.cs index 69b450a360..ab40e27fc6 100644 --- a/src/Stratis.Bitcoin.Features.PoA.Tests/VotingManagerTests.cs +++ b/src/Stratis.Bitcoin.Features.PoA.Tests/VotingManagerTests.cs @@ -24,8 +24,8 @@ public VotingManagerTests() this.changesApplied = new List(); this.changesReverted = new List(); - this.resultExecutorMock.Setup(x => x.ApplyChange(It.IsAny())).Callback((VotingData data) => this.changesApplied.Add(data)); - this.resultExecutorMock.Setup(x => x.RevertChange(It.IsAny())).Callback((VotingData data) => this.changesReverted.Add(data)); + this.resultExecutorMock.Setup(x => x.ApplyChange(It.IsAny(), It.IsAny())).Callback((VotingData data, int _) => this.changesApplied.Add(data)); + this.resultExecutorMock.Setup(x => x.RevertChange(It.IsAny(), It.IsAny())).Callback((VotingData data, int _) => this.changesReverted.Add(data)); } [Fact] diff --git a/src/Stratis.Bitcoin.Features.PoA/PoAFeature.cs b/src/Stratis.Bitcoin.Features.PoA/PoAFeature.cs index 08c469ff2f..1afc87c96d 100644 --- a/src/Stratis.Bitcoin.Features.PoA/PoAFeature.cs +++ b/src/Stratis.Bitcoin.Features.PoA/PoAFeature.cs @@ -137,7 +137,7 @@ public override Task InitializeAsync() } this.federationManager.Initialize(); - this.whitelistedHashesRepository.Initialize(); + this.whitelistedHashesRepository.Initialize(this.votingManager); if (!this.votingManager.Synchronize(this.chainIndexer.Tip)) throw new System.OperationCanceledException(); diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/PollResultExecutor.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/PollResultExecutor.cs index 2e10360f99..5545699e8f 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/PollResultExecutor.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/PollResultExecutor.cs @@ -8,11 +8,13 @@ public interface IPollResultExecutor { /// Applies effect of . /// See . - void ApplyChange(VotingData data); + /// The height from which the change should take effect. + void ApplyChange(VotingData data, int executionHeight); /// Reverts effect of . /// See . - void RevertChange(VotingData data); + /// The height from which the change should take effect. + void RevertChange(VotingData data, int executionHeight); /// Converts to a human readable format. /// See . @@ -40,7 +42,7 @@ public PollResultExecutor(IFederationManager federationManager, ILoggerFactory l } /// - public void ApplyChange(VotingData data) + public void ApplyChange(VotingData data, int executionHeight) { switch (data.Key) { @@ -53,17 +55,17 @@ public void ApplyChange(VotingData data) break; case VoteKey.WhitelistHash: - this.AddHash(data.Data); + this.AddHash(data.Data, executionHeight); break; case VoteKey.RemoveHash: - this.RemoveHash(data.Data); + this.RemoveHash(data.Data, executionHeight); break; } } /// - public void RevertChange(VotingData data) + public void RevertChange(VotingData data, int executionHeight) { switch (data.Key) { @@ -76,11 +78,11 @@ public void RevertChange(VotingData data) break; case VoteKey.WhitelistHash: - this.RemoveHash(data.Data); + this.RemoveHash(data.Data, executionHeight); break; case VoteKey.RemoveHash: - this.AddHash(data.Data); + this.AddHash(data.Data, executionHeight); break; } } @@ -118,13 +120,13 @@ public void RemoveFederationMember(byte[] federationMemberBytes) this.federationManager.RemoveFederationMember(federationMember); } - private void AddHash(byte[] hashBytes) + private void AddHash(byte[] hashBytes, int executionHeight) { try { var hash = new uint256(hashBytes); - this.whitelistedHashesRepository.AddHash(hash); + this.whitelistedHashesRepository.AddHash(hash, executionHeight); } catch (FormatException e) { @@ -132,13 +134,13 @@ private void AddHash(byte[] hashBytes) } } - private void RemoveHash(byte[] hashBytes) + private void RemoveHash(byte[] hashBytes, int executionHeight) { try { var hash = new uint256(hashBytes); - this.whitelistedHashesRepository.RemoveHash(hash); + this.whitelistedHashesRepository.RemoveHash(hash, executionHeight); } catch (FormatException e) { @@ -146,4 +148,4 @@ private void RemoveHash(byte[] hashBytes) } } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs index 7d5e8044b2..1b7497a04e 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs @@ -530,6 +530,9 @@ private void ProcessBlock(PollsRepository.Transaction transaction, ChainedHeader lock (this.locker) { this.signals.Publish(new VotingManagerProcessBlock(chBlock, transaction)); + + foreach (Poll poll in this.polls.GetPollsToExecuteOrExpire(chBlock.ChainedHeader.Height).OrderBy(p => p.Id)) + foreach (Poll poll in this.polls.GetPollsToExecuteOrExpire(chBlock.ChainedHeader.Height)) { if (!poll.IsApproved) @@ -544,7 +547,7 @@ private void ProcessBlock(PollsRepository.Transaction transaction, ChainedHeader else { this.logger.LogDebug("Applying poll '{0}'.", poll); - this.pollResultExecutor.ApplyChange(poll.VotingData); + this.pollResultExecutor.ApplyChange(poll.VotingData, chBlock.ChainedHeader.Height); this.polls.AdjustPoll(poll, poll => poll.PollExecutedBlockData = new HashHeightPair(chBlock.ChainedHeader)); transaction.UpdatePoll(poll); @@ -689,16 +692,22 @@ private void UnProcessBlock(PollsRepository.Transaction transaction, ChainedHead { lock (this.locker) { - foreach (Poll poll in this.polls.Where(x => !x.IsPending && x.PollExecutedBlockData?.Hash == chBlock.ChainedHeader.HashBlock).ToList()) + foreach (Poll poll in this.polls + .Where(x => !x.IsPending && x.PollExecutedBlockData?.Hash == chBlock.ChainedHeader.HashBlock) + .OrderByDescending(p => p.Id) + .ToList()) { this.logger.LogDebug("Reverting poll execution '{0}'.", poll); - this.pollResultExecutor.RevertChange(poll.VotingData); + this.pollResultExecutor.RevertChange(poll.VotingData, chBlock.ChainedHeader.Height); this.polls.AdjustPoll(poll, poll => poll.PollExecutedBlockData = null); transaction.UpdatePoll(poll); } - foreach (Poll poll in this.polls.Where(x => x.IsExpired && !PollsRepository.IsPollExpiredAt(x, chBlock.ChainedHeader.Height - 1, this.network as PoANetwork)).ToList()) + foreach (Poll poll in this.polls + .Where(x => x.IsExpired && !PollsRepository.IsPollExpiredAt(x, chBlock.ChainedHeader.Height - 1, this.network as PoANetwork)) + .OrderByDescending(p => p.Id) + .ToList()) { this.logger.LogDebug("Reverting poll expiry '{0}'.", poll); diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/WhitelistedHashesRepository.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/WhitelistedHashesRepository.cs index cd8ac70124..435f27410a 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/WhitelistedHashesRepository.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/WhitelistedHashesRepository.cs @@ -1,93 +1,163 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.Logging; using NBitcoin; -using Stratis.Bitcoin.Persistence; +using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.PoA.Voting { public class WhitelistedHashesRepository : IWhitelistedHashesRepository { - private const string dbKey = "hashesList"; - - private readonly IKeyValueRepository kvRepository; - /// Protects access to . private readonly object locker; private readonly ILogger logger; - private List whitelistedHashes; + private readonly PoAConsensusOptions poaConsensusOptions; - public WhitelistedHashesRepository(ILoggerFactory loggerFactory, IKeyValueRepository kvRepository) + // Dictionary of hash histories. Even list entries are additions and odd entries are removals. + private Dictionary whitelistedHashes; + + public WhitelistedHashesRepository(ILoggerFactory loggerFactory, Network network) { - this.kvRepository = kvRepository; this.locker = new object(); this.logger = loggerFactory.CreateLogger(this.GetType().FullName); + this.poaConsensusOptions = network.Consensus.Options as PoAConsensusOptions; + } - // Load this before initialize to ensure its available to when the Mempool feature initializes. - lock (this.locker) + public class PollComparer : IComparer<(int height, int id)> + { + public int Compare((int height, int id) poll1, (int height, int id) poll2) { - this.whitelistedHashes = this.kvRepository.LoadValueJson>(dbKey) ?? new List(); + int cmp = poll1.height.CompareTo(poll2.height); + if (cmp != 0) + return cmp; + + return poll1.id.CompareTo(poll2.id); } } - public void Initialize() + static PollComparer pollComparer = new PollComparer(); + + private void GetWhitelistedHashesFromExecutedPolls(VotingManager votingManager) { + lock (this.locker) + { + var federation = new List(this.poaConsensusOptions.GenesisFederationMembers); + + IEnumerable executedPolls = votingManager.GetExecutedPolls().WhitelistPolls(); + foreach (Poll poll in executedPolls.OrderBy(a => (a.PollExecutedBlockData.Height, a.Id), pollComparer)) + { + var hash = new uint256(poll.VotingData.Data); + + if (poll.VotingData.Key == VoteKey.WhitelistHash) + { + this.AddHash(hash, poll.PollExecutedBlockData.Height); + } + else if (poll.VotingData.Key == VoteKey.RemoveHash) + { + this.RemoveHash(hash, poll.PollExecutedBlockData.Height); + } + } + } } - private void SaveHashes() + public void Initialize(VotingManager votingManager) { + // TODO: Must call Initialize before the Mempool rules try to use this class. lock (this.locker) { - this.kvRepository.SaveValueJson(dbKey, this.whitelistedHashes); + this.whitelistedHashes = new Dictionary(); + this.GetWhitelistedHashesFromExecutedPolls(votingManager); } } - public void AddHash(uint256 hash) + public void AddHash(uint256 hash, int executionHeight) { lock (this.locker) { - if (this.whitelistedHashes.Contains(hash)) + // Retrieve the whitelist history for this hash. + if (!this.whitelistedHashes.TryGetValue(hash, out int[] history)) { - this.logger.LogTrace("(-)[ALREADY_EXISTS]"); + this.whitelistedHashes[hash] = new int[] { executionHeight }; return; } - this.whitelistedHashes.Add(hash); + // Keep all history up to and including the executionHeight. + int keep = BinarySearch.BinaryFindFirst((k) => k == history.Length || history[k] > executionHeight, 0, history.Length + 1); + Array.Resize(ref history, keep | 1); + this.whitelistedHashes[hash] = history; + + // If the history is an even length then add the addition height to signify addition. + if ((keep % 2) == 0) + { + // Add an even indexed entry to signify an addition. + history[keep] = executionHeight; + return; + } + + this.logger.LogTrace("(-)[HASH_ALREADY_EXISTS]"); + return; + } + } + + public void RemoveHash(uint256 hash, int executionHeight) + { + lock (this.locker) + { + // Retrieve the whitelist history for this hash. + if (this.whitelistedHashes.TryGetValue(hash, out int[] history)) + { + // Keep all history up to and including the executionHeight. + int keep = BinarySearch.BinaryFindFirst((k) => k == history.Length || history[k] >= executionHeight, 0, history.Length + 1); + Array.Resize(ref history, (keep + 1) & ~1); + this.whitelistedHashes[hash] = history; + + // If the history is an odd length then add the removal height to signify removal. + if ((keep % 2) != 0) + { + history[keep] = executionHeight; + return; + } + } - this.SaveHashes(); + this.logger.LogTrace("(-)[HASH_DOESNT_EXIST]"); + return; } } - public void RemoveHash(uint256 hash) + private bool ExistsHash(uint256 hash, int blockHeight) { lock (this.locker) { - bool removed = this.whitelistedHashes.Remove(hash); + // Retrieve the whitelist history for this hash. + if (!this.whitelistedHashes.TryGetValue(hash, out int[] history)) + return false; - if (removed) - this.SaveHashes(); + int keep = BinarySearch.BinaryFindFirst((k) => k == history.Length || history[k] > blockHeight, 0, history.Length + 1); + return (keep % 2) != 0; } } - public List GetHashes() + public List GetHashes(int blockHeight = int.MaxValue) { lock (this.locker) { - return new List(this.whitelistedHashes); + return this.whitelistedHashes.Where(k => ExistsHash(k.Key, blockHeight)).Select(k => k.Key).ToList(); } } } public interface IWhitelistedHashesRepository { - void AddHash(uint256 hash); + void AddHash(uint256 hash, int executionHeight); - void RemoveHash(uint256 hash); + void RemoveHash(uint256 hash, int executionHeight); - List GetHashes(); + List GetHashes(int blockHeight = int.MaxValue); - void Initialize(); + void Initialize(VotingManager votingManager); } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/Consensus/Rules/AllowedCodeHashLogicTests.cs b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/Consensus/Rules/AllowedCodeHashLogicTests.cs index 3a6f0b1940..ad2b9d509a 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/Consensus/Rules/AllowedCodeHashLogicTests.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/Consensus/Rules/AllowedCodeHashLogicTests.cs @@ -30,15 +30,15 @@ public void Should_Allow_Code_With_Valid_Hash() byte[] hash = HashHelper.Keccak256(code); this.hashingStrategy.Setup(h => h.Hash(code)).Returns(hash); - this.hashChecker.Setup(h => h.CheckHashWhitelisted(hash)).Returns(true); + this.hashChecker.Setup(h => h.CheckHashWhitelisted(hash, 0)).Returns(true); - var tx = new ContractTxData(1, 1000, (Gas) 10000, code); + var tx = new ContractTxData(1, 1000, (Gas)10000, code); var sut = new AllowedCodeHashLogic(this.hashChecker.Object, this.hashingStrategy.Object); sut.CheckContractTransaction(tx, 0); - this.hashChecker.Verify(h => h.CheckHashWhitelisted(hash), Times.Once); + this.hashChecker.Verify(h => h.CheckHashWhitelisted(hash, 0), Times.Once); } [Fact] @@ -49,7 +49,7 @@ public void Should_Throw_ConsensusErrorException_If_Hash_Not_Allowed() byte[] hash = HashHelper.Keccak256(code); this.hashingStrategy.Setup(h => h.Hash(code)).Returns(hash); - this.hashChecker.Setup(h => h.CheckHashWhitelisted(hash)).Returns(false); + this.hashChecker.Setup(h => h.CheckHashWhitelisted(hash, 0)).Returns(false); var sut = new AllowedCodeHashLogic(this.hashChecker.Object, this.hashingStrategy.Object); @@ -57,7 +57,7 @@ public void Should_Throw_ConsensusErrorException_If_Hash_Not_Allowed() Assert.Throws(() => sut.CheckContractTransaction(tx, 0)); - this.hashChecker.Verify(h => h.CheckHashWhitelisted(hash), Times.Once); + this.hashChecker.Verify(h => h.CheckHashWhitelisted(hash, 0), Times.Once); } [Fact] @@ -70,7 +70,7 @@ public void Should_Not_Validate_ContractCall() sut.CheckContractTransaction(callTx, 0); this.hashingStrategy.Verify(h => h.Hash(It.IsAny()), Times.Never); - this.hashChecker.Verify(h => h.CheckHashWhitelisted(It.IsAny()), Times.Never); + this.hashChecker.Verify(h => h.CheckHashWhitelisted(It.IsAny(), It.IsAny()), Times.Never); } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/WhitelistHashCheckerTests.cs b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/WhitelistHashCheckerTests.cs index 693f26e1c3..b0cf242fa5 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/WhitelistHashCheckerTests.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/WhitelistHashCheckerTests.cs @@ -16,11 +16,11 @@ public void Should_Return_False_If_Hash_Not_In_Whitelist() var repository = new Mock(); - repository.Setup(r => r.GetHashes()).Returns(new List()); + repository.Setup(r => r.GetHashes(It.IsAny())).Returns(new List()); var strategy = new WhitelistedHashChecker(repository.Object); - Assert.False(strategy.CheckHashWhitelisted(hash)); + Assert.False(strategy.CheckHashWhitelisted(hash, 0)); } [Fact] @@ -30,11 +30,11 @@ public void Should_Return_True_If_Hash_In_Whitelist() var repository = new Mock(); - repository.Setup(r => r.GetHashes()).Returns(new List { new uint256(hash) }); + repository.Setup(r => r.GetHashes(It.IsAny())).Returns(new List { new uint256(hash) }); var strategy = new WhitelistedHashChecker(repository.Object); - Assert.True(strategy.CheckHashWhitelisted(hash)); + Assert.True(strategy.CheckHashWhitelisted(hash, 0)); } [Fact] @@ -42,14 +42,14 @@ public void Should_Return_False_Invalid_uint256() { var repository = new Mock(); - repository.Setup(r => r.GetHashes()).Returns(new List()); + repository.Setup(r => r.GetHashes(It.IsAny())).Returns(new List()); var strategy = new WhitelistedHashChecker(repository.Object); // uint256 must be 256 bytes wide var invalidUint256Bytes = new byte[] { }; - Assert.False(strategy.CheckHashWhitelisted(invalidUint256Bytes)); + Assert.False(strategy.CheckHashWhitelisted(invalidUint256Bytes, 0)); } } } \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/PoA/Rules/AllowedCodeHashLogic.cs b/src/Stratis.Bitcoin.Features.SmartContracts/PoA/Rules/AllowedCodeHashLogic.cs index 2e3dcba4ca..0c3ad0eb1e 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/PoA/Rules/AllowedCodeHashLogic.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/PoA/Rules/AllowedCodeHashLogic.cs @@ -27,7 +27,7 @@ public void CheckContractTransaction(ContractTxData txData, Money suppliedBudget byte[] hashedCode = this.hashingStrategy.Hash(txData.ContractExecutionCode); - if (!this.whitelistedHashChecker.CheckHashWhitelisted(hashedCode)) + if (!this.whitelistedHashChecker.CheckHashWhitelisted(hashedCode, blockHeight)) { ThrowInvalidCode(); } diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/PoA/WhitelistedHashChecker.cs b/src/Stratis.Bitcoin.Features.SmartContracts/PoA/WhitelistedHashChecker.cs index 5372192786..81437c2f4d 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/PoA/WhitelistedHashChecker.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/PoA/WhitelistedHashChecker.cs @@ -21,7 +21,7 @@ public WhitelistedHashChecker(IWhitelistedHashesRepository whitelistedHashesRepo /// /// The bytes of the hash to check. /// True if the hash was found in the whitelisted hashes repository. - public bool CheckHashWhitelisted(byte[] hash) + public bool CheckHashWhitelisted(byte[] hash, int blockHeight) { if (hash.Length != 32) { @@ -29,7 +29,7 @@ public bool CheckHashWhitelisted(byte[] hash) return false; } - List allowedHashes = this.whitelistedHashesRepository.GetHashes(); + List allowedHashes = this.whitelistedHashesRepository.GetHashes(blockHeight); // Now that we've checked the width of the byte array, we don't expect the uint256 constructor to throw any exceptions. var hash256 = new uint256(hash); @@ -40,6 +40,6 @@ public bool CheckHashWhitelisted(byte[] hash) public interface IWhitelistedHashChecker { - bool CheckHashWhitelisted(byte[] hash); + bool CheckHashWhitelisted(byte[] hash, int blockHeight); } } \ No newline at end of file diff --git a/src/Stratis.SmartContracts.IntegrationTests/WhitelistNodeExtensions.cs b/src/Stratis.SmartContracts.IntegrationTests/WhitelistNodeExtensions.cs index a3caacf7a4..db0b16d828 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/WhitelistNodeExtensions.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/WhitelistNodeExtensions.cs @@ -15,7 +15,7 @@ public static void WhitelistCode(this IMockChain chain, byte[] code) { var hasher = node.CoreNode.FullNode.NodeService(); var hash = new uint256(hasher.Hash(code)); - node.CoreNode.FullNode.NodeService().AddHash(hash); + node.CoreNode.FullNode.NodeService().AddHash(hash, 0); } } From 602664ea2f7efd5d43cc4c56ecee504a4b952596 Mon Sep 17 00:00:00 2001 From: zeptin Date: Wed, 23 Nov 2022 09:58:02 +0200 Subject: [PATCH 41/44] Update BTC checkpoints (#1096) * Update BTC checkpoints * Fix invalid appsettings field * Remove unused solution folder * Fix tests * Fix test * Amend non-critical test (cherry picked from commit 6b60494518038c89e4efb5c86bc21e93d31f4019) --- src/NBitcoin.Tests/NetworkTests.cs | 8 ++++---- src/Stratis.Bitcoin.Features.Api/appsettings.json | 4 +++- src/Stratis.Bitcoin.Networks/BitcoinMain.cs | 7 ++++--- src/Stratis.Bitcoin.Networks/BitcoinTest.cs | 5 +++-- src/Stratis.Bitcoin.Tests/Consensus/CheckpointsTest.cs | 9 ++++++--- src/Stratis.Features.Unity3dApi/appsettings.json | 6 ++++-- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/NBitcoin.Tests/NetworkTests.cs b/src/NBitcoin.Tests/NetworkTests.cs index cffc179c20..cc637de0ed 100644 --- a/src/NBitcoin.Tests/NetworkTests.cs +++ b/src/NBitcoin.Tests/NetworkTests.cs @@ -79,7 +79,7 @@ public void ReadMagicByteWithFirstByteDuplicated() [Trait("UnitTest", "UnitTest")] public void BitcoinMainnetIsInitializedCorrectly() { - Assert.Equal(17, this.networkMain.Checkpoints.Count); + Assert.Equal(18, this.networkMain.Checkpoints.Count); Assert.Equal(6, this.networkMain.DNSSeeds.Count); Assert.Equal(512, this.networkMain.SeedNodes.Count); @@ -143,7 +143,7 @@ public void BitcoinMainnetIsInitializedCorrectly() Assert.Equal(Utils.UnixTimeToDateTime(1510704000), this.networkMain.Consensus.BIP9Deployments[BitcoinBIP9Deployments.Segwit].Timeout); Assert.Equal(0, this.networkMain.Consensus.CoinType); Assert.False(this.networkMain.Consensus.IsProofOfStake); - Assert.Equal(new uint256("0x0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee"), this.networkMain.Consensus.DefaultAssumeValid); + Assert.Equal(new uint256("0x00000000000000000003e1d91b245eb32787afb10afe49b61621375361221c38"), this.networkMain.Consensus.DefaultAssumeValid); Assert.Equal(100, this.networkMain.Consensus.CoinbaseMaturity); Assert.Equal(0, this.networkMain.Consensus.PremineReward); Assert.Equal(0, this.networkMain.Consensus.PremineHeight); @@ -163,7 +163,7 @@ public void BitcoinTestnetIsInitializedCorrectly() { Network network = KnownNetworks.TestNet; - Assert.Equal(13, network.Checkpoints.Count); + Assert.Equal(14, network.Checkpoints.Count); Assert.Equal(3, network.DNSSeeds.Count); Assert.Empty(network.SeedNodes); @@ -224,7 +224,7 @@ public void BitcoinTestnetIsInitializedCorrectly() Assert.Equal(Utils.UnixTimeToDateTime(1493596800), network.Consensus.BIP9Deployments[BitcoinBIP9Deployments.Segwit].Timeout); Assert.Equal(1, network.Consensus.CoinType); Assert.False(network.Consensus.IsProofOfStake); - Assert.Equal(new uint256("0x0000000000000037a8cd3e06cd5edbfe9dd1dbcc5dacab279376ef7cfc2b4c75"), network.Consensus.DefaultAssumeValid); + Assert.Equal(new uint256("000000000000075566fba6ca27a2e4c8b33c7763f8a5f917b231b0d88c743af8"), network.Consensus.DefaultAssumeValid); Assert.Equal(100, network.Consensus.CoinbaseMaturity); Assert.Equal(0, network.Consensus.PremineReward); Assert.Equal(0, network.Consensus.PremineHeight); diff --git a/src/Stratis.Bitcoin.Features.Api/appsettings.json b/src/Stratis.Bitcoin.Features.Api/appsettings.json index 3293d7acd5..47f2da68a8 100644 --- a/src/Stratis.Bitcoin.Features.Api/appsettings.json +++ b/src/Stratis.Bitcoin.Features.Api/appsettings.json @@ -1,10 +1,12 @@ { "Logging": { - "IncludeScopes": false, "LogLevel": { "Default": "Information", "System": "Information", "Microsoft": "Information" + }, + "Console": { + "IncludeScopes": false } } } diff --git a/src/Stratis.Bitcoin.Networks/BitcoinMain.cs b/src/Stratis.Bitcoin.Networks/BitcoinMain.cs index f10c6a1339..62615372ed 100644 --- a/src/Stratis.Bitcoin.Networks/BitcoinMain.cs +++ b/src/Stratis.Bitcoin.Networks/BitcoinMain.cs @@ -78,7 +78,7 @@ public BitcoinMain() bip34Hash: new uint256("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"), minerConfirmationWindow: 2016, // nPowTargetTimespan / nPowTargetSpacing maxReorgLength: 0, - defaultAssumeValid: new uint256("0x0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee"), // 563378 + defaultAssumeValid: new uint256("0x00000000000000000003e1d91b245eb32787afb10afe49b61621375361221c38"), // 760000 maxMoney: 21000000 * Money.COIN, coinbaseMaturity: 100, premineHeight: 0, @@ -135,8 +135,9 @@ public BitcoinMain() { 295000, new CheckpointInfo(new uint256("0x00000000000000004d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983"))}, { 486000, new CheckpointInfo(new uint256("0x000000000000000000a2a8104d61651f76c666b70754d6e9346176385f7afa24"))}, { 491800, new CheckpointInfo(new uint256("0x000000000000000000d80de1f855902b50941bc3a3d0f71064d9613fd3943dc4"))}, - { 550000, new CheckpointInfo(new uint256("0x000000000000000000223b7a2298fb1c6c75fb0efc28a4c56853ff4112ec6bc9"))}, // 14-11-2018, - { 610000, new CheckpointInfo(new uint256("0x0000000000000000000a6f607f74db48dae0a94022c10354536394c17672b7f7"))} // 27-12-2019 + { 550000, new CheckpointInfo(new uint256("0x000000000000000000223b7a2298fb1c6c75fb0efc28a4c56853ff4112ec6bc9"))}, // 2018-11-14 + { 610000, new CheckpointInfo(new uint256("0x0000000000000000000a6f607f74db48dae0a94022c10354536394c17672b7f7"))}, // 2019-12-27 + { 760000, new CheckpointInfo(new uint256("0x00000000000000000003e1d91b245eb32787afb10afe49b61621375361221c38"))} // 2022-10-23 }; this.DNSSeeds = new List diff --git a/src/Stratis.Bitcoin.Networks/BitcoinTest.cs b/src/Stratis.Bitcoin.Networks/BitcoinTest.cs index 686924a01d..e20597dd45 100644 --- a/src/Stratis.Bitcoin.Networks/BitcoinTest.cs +++ b/src/Stratis.Bitcoin.Networks/BitcoinTest.cs @@ -65,7 +65,7 @@ public BitcoinTest() bip34Hash: new uint256("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"), minerConfirmationWindow: 2016, maxReorgLength: 0, - defaultAssumeValid: new uint256("0x0000000000000037a8cd3e06cd5edbfe9dd1dbcc5dacab279376ef7cfc2b4c75"), // 1354312 + defaultAssumeValid: new uint256("0x000000000000075566fba6ca27a2e4c8b33c7763f8a5f917b231b0d88c743af8"), // 2400000 maxMoney: 21000000 * Money.COIN, coinbaseMaturity: 100, premineHeight: 0, @@ -113,7 +113,8 @@ public BitcoinTest() { 800_000, new CheckpointInfo(new uint256("0000000000209b091d6519187be7c2ee205293f25f9f503f90027e25abf8b503")) }, { 1_000_000, new CheckpointInfo(new uint256("0000000000478e259a3eda2fafbeeb0106626f946347955e99278fe6cc848414")) }, { 1_210_000, new CheckpointInfo(new uint256("00000000461201277cf8c635fc10d042d6f0a7eaa57f6c9e8c099b9e0dbc46dc")) }, - { 1_400_000, new CheckpointInfo(new uint256("000000000000fce208da3e3b8afcc369835926caa44044e9c2f0caa48c8eba0f")) } // 22-08-2018 + { 1_400_000, new CheckpointInfo(new uint256("000000000000fce208da3e3b8afcc369835926caa44044e9c2f0caa48c8eba0f")) }, // 2018-08-22 + { 2_400_000, new CheckpointInfo(new uint256("000000000000075566fba6ca27a2e4c8b33c7763f8a5f917b231b0d88c743af8")) } // 2022-11-01 }; this.DNSSeeds = new List diff --git a/src/Stratis.Bitcoin.Tests/Consensus/CheckpointsTest.cs b/src/Stratis.Bitcoin.Tests/Consensus/CheckpointsTest.cs index 78fb48357f..5ec6ee9d17 100644 --- a/src/Stratis.Bitcoin.Tests/Consensus/CheckpointsTest.cs +++ b/src/Stratis.Bitcoin.Tests/Consensus/CheckpointsTest.cs @@ -44,7 +44,7 @@ public void GetLastCheckPointHeight_BitcoinMainnet_ReturnsLastCheckPointHeight() int result = checkpoints.GetLastCheckpointHeight(); - Assert.Equal(610000, result); + Assert.Equal(760000, result); } [Fact] @@ -54,7 +54,7 @@ public void GetLastCheckPointHeight_BitcoinTestnet_ReturnsLastCheckPointHeight() int result = checkpoints.GetLastCheckpointHeight(); - Assert.Equal(1400000, result); + Assert.Equal(2400000, result); } [Fact] @@ -99,7 +99,7 @@ public void GetLastCheckPointHeight_CheckpointsEnabledAfterLoad_RetrievesCheckpo consensusSettings.UseCheckpoints = true; result = checkpoints.GetLastCheckpointHeight(); - Assert.Equal(610000, result); + Assert.Equal(760000, result); } [Fact] @@ -224,6 +224,9 @@ public void VerifyCheckpoints_BitcoinMainnet() { 295000, new CheckpointInfo(new uint256("0x00000000000000004d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983")) }, { 486000, new CheckpointInfo(new uint256("0x000000000000000000a2a8104d61651f76c666b70754d6e9346176385f7afa24")) }, { 491800, new CheckpointInfo(new uint256("0x000000000000000000d80de1f855902b50941bc3a3d0f71064d9613fd3943dc4")) }, + { 550000, new CheckpointInfo(new uint256("0x000000000000000000223b7a2298fb1c6c75fb0efc28a4c56853ff4112ec6bc9")) }, + { 610000, new CheckpointInfo(new uint256("0x0000000000000000000a6f607f74db48dae0a94022c10354536394c17672b7f7")) }, + { 760000, new CheckpointInfo(new uint256("0x00000000000000000003e1d91b245eb32787afb10afe49b61621375361221c38")) }, }; var checkpoints = new Checkpoints(this.network, new ConsensusSettings(NodeSettings.Default(this.network)) { UseCheckpoints = true }); diff --git a/src/Stratis.Features.Unity3dApi/appsettings.json b/src/Stratis.Features.Unity3dApi/appsettings.json index 4e0e03cdca..3de0b0a990 100644 --- a/src/Stratis.Features.Unity3dApi/appsettings.json +++ b/src/Stratis.Features.Unity3dApi/appsettings.json @@ -1,10 +1,12 @@ { "Logging": { - "IncludeScopes": false, "LogLevel": { "Default": "Information", "System": "Information", "Microsoft": "Information" + }, + "Console": { + "IncludeScopes": false } } -} \ No newline at end of file +} From 3b5080138a844ff292251e1b7ca5517795e761fa Mon Sep 17 00:00:00 2001 From: zeptin Date: Sun, 4 Dec 2022 19:12:13 +0200 Subject: [PATCH 42/44] Add -blockmintxfee setting (#1097) * Add dumpprivkey RPC command * Revert superfluous change * Add additional sanity test * Fix test * Add -blockmintxfee setting * Fix settings property usage (cherry picked from commit bdc9f2db52f51036635fc4c09a99c8b9921edb4e) --- .../BlockDefinitionOptions.cs | 8 +++++--- src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs | 5 ++++- .../BlockBufferGeneratorTests.cs | 4 ++-- .../BlockBufferGenerator.cs | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Miner/BlockDefinitionOptions.cs b/src/Stratis.Bitcoin.Features.Miner/BlockDefinitionOptions.cs index 6f02e00cb2..3fab99c47c 100644 --- a/src/Stratis.Bitcoin.Features.Miner/BlockDefinitionOptions.cs +++ b/src/Stratis.Bitcoin.Features.Miner/BlockDefinitionOptions.cs @@ -20,11 +20,11 @@ public sealed class BlockDefinitionOptions /// Minimum fee rate for transactions to be included in blocks created by miner. public FeeRate BlockMinFeeRate { get; private set; } - public BlockDefinitionOptions(uint blockMaxWeight, uint blockMaxSize) + public BlockDefinitionOptions(uint blockMaxWeight, uint blockMaxSize, uint blockMinTxFee) { this.BlockMaxWeight = blockMaxWeight; this.BlockMaxSize = blockMaxSize; - this.BlockMinFeeRate = new FeeRate(PowMining.DefaultBlockMinTxFee); // TODO: Where should this be set, really? + this.BlockMinFeeRate = new FeeRate(blockMinTxFee); } /// @@ -39,7 +39,9 @@ public BlockDefinitionOptions RestrictForNetwork(Network network) this.BlockMaxWeight = Math.Max(minAllowedBlockWeight, Math.Min(network.Consensus.Options.MaxBlockWeight, this.BlockMaxWeight)); this.BlockMaxSize = Math.Max(MinBlockSize, Math.Min(network.Consensus.Options.MaxBlockSerializedSize, this.BlockMaxSize)); + // Note: The minimum fee rate is not a consensus parameter; miners can elect to include low or zero-fee transactions if they wish. + return this; } } -} \ No newline at end of file +} diff --git a/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs b/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs index 317171e175..01386b2512 100644 --- a/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs +++ b/src/Stratis.Bitcoin.Features.Miner/MinerSettings.cs @@ -74,6 +74,9 @@ public class MinerSettings : BaseSettings private uint BlockMaxWeight { get { return this.blockMaxWeight ?? this.nodeSettings.Network.Consensus.Options.MaxBlockWeight; } set { this.blockMaxWeight = value; } } private uint? blockMaxWeight = null; + [CommandLineOption("blockmintxfee", "Set lowest fee rate (in BTC/kvB) for transactions to be included in block creation.")] + public uint BlockMinTxFee { get; set; } = PowMining.DefaultBlockMinTxFee; + /// /// Settings for . /// @@ -85,7 +88,7 @@ public class MinerSettings : BaseSettings /// The node configuration. public MinerSettings(NodeSettings nodeSettings) : base(nodeSettings) { - this.BlockDefinitionOptions = new BlockDefinitionOptions(this.BlockMaxWeight, this.BlockMaxSize).RestrictForNetwork(nodeSettings.Network); + this.BlockDefinitionOptions = new BlockDefinitionOptions(this.BlockMaxWeight, this.BlockMaxSize, this.BlockMinTxFee).RestrictForNetwork(nodeSettings.Network); if (!this.Mine) this.MineAddress = null; diff --git a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/BlockBufferGeneratorTests.cs b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/BlockBufferGeneratorTests.cs index 30ccdad257..21c4ec444f 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts.Tests/BlockBufferGeneratorTests.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts.Tests/BlockBufferGeneratorTests.cs @@ -10,11 +10,11 @@ public class BlockBufferGeneratorTests public void Buffer_50Kb_For_1MB_BlockSize() { var network = new SmartContractsRegTest(); - var optionsFromNetwork = new BlockDefinitionOptions(network.Consensus.Options.MaxBlockWeight, network.Consensus.Options.MaxBlockBaseSize); + var optionsFromNetwork = new BlockDefinitionOptions(network.Consensus.Options.MaxBlockWeight, network.Consensus.Options.MaxBlockBaseSize, PowMining.DefaultBlockMinTxFee); BlockDefinitionOptions newOptions = new BlockBufferGenerator().GetOptionsWithBuffer(optionsFromNetwork); Assert.Equal((uint)950_000, newOptions.BlockMaxWeight); Assert.Equal((uint)950_000, newOptions.BlockMaxSize); } } -} \ No newline at end of file +} diff --git a/src/Stratis.Bitcoin.Features.SmartContracts/BlockBufferGenerator.cs b/src/Stratis.Bitcoin.Features.SmartContracts/BlockBufferGenerator.cs index ea85aa59fe..ed6f0f0598 100644 --- a/src/Stratis.Bitcoin.Features.SmartContracts/BlockBufferGenerator.cs +++ b/src/Stratis.Bitcoin.Features.SmartContracts/BlockBufferGenerator.cs @@ -7,7 +7,7 @@ public class BlockBufferGenerator : IBlockBufferGenerator public BlockDefinitionOptions GetOptionsWithBuffer(BlockDefinitionOptions options) { uint percentageToBuild = 95; // For 1MB blocks, 50 KB reserved for generated transactions / txouts - return new BlockDefinitionOptions(options.BlockMaxSize * percentageToBuild / 100, options.BlockMaxWeight * percentageToBuild / 100); + return new BlockDefinitionOptions(options.BlockMaxSize * percentageToBuild / 100, options.BlockMaxWeight * percentageToBuild / 100, (uint)options.BlockMinFeeRate.FeePerK.Satoshi); } } } From d11fc478d62a6c1a912aa9f0a5f566005d0b3668 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 31 Jan 2023 17:31:50 +1100 Subject: [PATCH 43/44] Fix merge --- src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs index 1b7497a04e..10eeb38841 100644 --- a/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs +++ b/src/Stratis.Bitcoin.Features.PoA/Voting/VotingManager.cs @@ -532,8 +532,6 @@ private void ProcessBlock(PollsRepository.Transaction transaction, ChainedHeader this.signals.Publish(new VotingManagerProcessBlock(chBlock, transaction)); foreach (Poll poll in this.polls.GetPollsToExecuteOrExpire(chBlock.ChainedHeader.Height).OrderBy(p => p.Id)) - - foreach (Poll poll in this.polls.GetPollsToExecuteOrExpire(chBlock.ChainedHeader.Height)) { if (!poll.IsApproved) { From 8b469591051e467946274d993991d007db66d57a Mon Sep 17 00:00:00 2001 From: quantumagi Date: Thu, 16 Feb 2023 21:43:11 +1100 Subject: [PATCH 44/44] Revert inadvertent changes --- src/Stratis.CirrusMinerD/Properties/launchSettings.json | 6 +----- src/Stratis.StraxD/Properties/launchSettings.json | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Stratis.CirrusMinerD/Properties/launchSettings.json b/src/Stratis.CirrusMinerD/Properties/launchSettings.json index 917be4c35c..7fefdba2b0 100644 --- a/src/Stratis.CirrusMinerD/Properties/launchSettings.json +++ b/src/Stratis.CirrusMinerD/Properties/launchSettings.json @@ -1,12 +1,8 @@ { "profiles": { - "Stratis.CirrusMinerD_Main": { - "commandName": "Project", - "commandLineArgs": "-mainchain -addressindex" - }, "Stratis.CirrusMinerD": { "commandName": "Project", - "commandLineArgs": "-sidechain" + "commandLineArgs": "-devmode=miner" }, "Stratis.CirrusMinerD TestNet": { "commandName": "Project", diff --git a/src/Stratis.StraxD/Properties/launchSettings.json b/src/Stratis.StraxD/Properties/launchSettings.json index 5789d2e9bf..b58a98efb9 100644 --- a/src/Stratis.StraxD/Properties/launchSettings.json +++ b/src/Stratis.StraxD/Properties/launchSettings.json @@ -1,8 +1,7 @@ { "profiles": { "Stratis.StraxD": { - "commandName": "Project", - "commandLineArgs": "-addressindex" + "commandName": "Project" }, "Stratis.StraxD Test": { "commandName": "Project",