Store Sapling and Orchard subtree roots in block index & add z_getsubtreesbyindex rpc

src/main.cpp

@@ -3670,6 +3670,81 @@ CCriticalSection& get_cs_main()
     return cs_main;
 }
 
+static bool AlignPersistedShieldedSubtreesForType(ShieldedType type, std::string& error)
+{
+    AssertLockHeld(cs_main);
+
+    if (pblocktree == nullptr) {
+        return true;
+    }
+
+    while (true) {
+        uint64_t latestIndex = 0;
+        if (!pblocktree->ReadLatestShieldedSubtreeIndex(type, latestIndex)) {
+            return true;
+        }
+
+        ShieldedSubtreeData subtree;
+        if (!pblocktree->ReadShieldedSubtree(type, latestIndex, subtree)) {
+            error = strprintf(
+                "missing persisted subtree metadata for type %d at index %llu",
+                static_cast<int>(type),
+                static_cast<unsigned long long>(latestIndex));
+            return false;
+        }
+
+        const int activeTipHeight = chainActive.Tip() ? chainActive.Tip()->nHeight : -1;
+        if (subtree.nHeight < 0) {
+            error = strprintf(
+                "persisted subtree metadata for type %d has invalid height %d",
+                static_cast<int>(type),
+                subtree.nHeight);
+            return false;
+        }
+
+        if (subtree.nHeight > activeTipHeight) {
+            if (!pblocktree->TruncateShieldedSubtrees(type, activeTipHeight)) {
+                error = strprintf(
+                    "failed to truncate persisted subtree metadata for type %d above active tip height %d",
+                    static_cast<int>(type),
+                    activeTipHeight);
+                return false;
+            }
+            continue;
+        }
+
+        const CBlockIndex* pindex = chainActive[subtree.nHeight];
+        if (pindex == nullptr) {
+            error = strprintf(
+                "active chain missing block at subtree completion height %d for type %d",
+                subtree.nHeight,
+                static_cast<int>(type));
+            return false;
+        }
+
+        if (pindex->GetBlockHash() != subtree.blockHash) {
+            const int truncateHeight = subtree.nHeight - 1;
+            if (!pblocktree->TruncateShieldedSubtrees(type, truncateHeight)) {
+                error = strprintf(
+                    "failed to rewind persisted subtree metadata for type %d from height %d",
+                    static_cast<int>(type),
+                    subtree.nHeight);
+                return false;
+            }
+            continue;
+        }
+
+        return true;
+    }
+}
+
+bool AlignPersistedShieldedSubtrees(std::string& error)
+{
+    AssertLockHeld(cs_main);
+    return AlignPersistedShieldedSubtreesForType(SAPLINGFRONTIER, error) &&
+           AlignPersistedShieldedSubtreesForType(ORCHARDFRONTIER, error);
+}
+
 /*****
  * @brief Apply the effects of this block (with given index) on the UTXO set represented by coins
  * @param block the block to add
@@ -3862,12 +3937,17 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
     OrchardMerkleFrontier orchard_frontier_tree;
     assert(view.GetOrchardFrontierAnchorAt(view.GetBestAnchor(ORCHARDFRONTIER), orchard_frontier_tree));
 
+    const bool persistShieldedSubtreeMetadata =
+        !fJustCheck && pcoinsTip != nullptr && view.GetBestBlock() == pcoinsTip->GetBestBlock();
+
     // Grab the consensus branch ID for this block and its parent
     auto consensusBranchId = CurrentEpochBranchId(pindex->nHeight, chainparams.GetConsensus());
     auto prevConsensusBranchId = CurrentEpochBranchId(pindex->nHeight - 1, chainparams.GetConsensus());
 
     size_t total_sapling_tx = 0;
     size_t total_orchard_tx = 0;
+    std::vector<std::pair<uint64_t, ShieldedSubtreeData>> completedSaplingSubtrees;
+    std::vector<std::pair<uint64_t, ShieldedSubtreeData>> completedOrchardSubtrees;
 
     CAmount chainSupplyDelta = 0;
     CAmount transparentValueDelta = 0;
@@ -4040,15 +4120,47 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
                 uint256 commitment = uint256::FromRawBytes(output.cmu());
                 sapling_tree.append(commitment);
             }
-            
+
             // Append to frontier tree using bundle
-            sapling_frontier_tree.AppendBundle(tx.GetSaplingBundle());
+            auto saplingResult = sapling_frontier_tree.AppendBundle(tx.GetSaplingBundle());
+            if (saplingResult.has_subtree_boundary) {
+                auto completedIndex = sapling_frontier_tree.current_subtree_index();
+                if (completedIndex == 0) {
+                    return state.DoS(
+                        100,
+                        error("ConnectBlock(): Sapling subtree boundary reached before subtree index advanced"),
+                        REJECT_INVALID,
+                        "bad-sapling-subtree-index");
+                }
+                completedSaplingSubtrees.emplace_back(
+                    completedIndex - 1,
+                    ShieldedSubtreeData(
+                        saplingResult.completed_subtree_root,
+                        pindex->nHeight,
+                        pindex->GetBlockHash()));
+            }
             total_sapling_tx += 1;
         }
 
         //Append Orchard Outputs to OrchardMerkleFrontier
         if (tx.GetOrchardBundle().IsPresent()) {
-            orchard_frontier_tree.AppendBundle(tx.GetOrchardBundle());
+            auto orchardResult = orchard_frontier_tree.AppendBundle(tx.GetOrchardBundle());
+            if (orchardResult.has_subtree_boundary) {
+                auto completedIndex = orchard_frontier_tree.current_subtree_index();
+                if (completedIndex == 0) {
+                    return state.DoS(
+                        100,
+                        error("ConnectBlock(): Orchard subtree boundary reached before subtree index advanced"),
+                        REJECT_INVALID,
+                        "bad-orchard-subtree-index");
+                }
+                completedOrchardSubtrees.emplace_back(
+                    completedIndex - 1,
+                    ShieldedSubtreeData(
+                        orchardResult.completed_subtree_root,
+                        pindex->nHeight,
+                        pindex->GetBlockHash()));
+            }
             total_orchard_tx += 1;
         }
 
@@ -4336,6 +4448,19 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
             return AbortNode(state, "Failed to write blockhash index");
     }
 
+    if (persistShieldedSubtreeMetadata) {
+        if (!completedSaplingSubtrees.empty()) {
+            if (!pblocktree->WriteShieldedSubtrees(SAPLINGFRONTIER, completedSaplingSubtrees)) {
+                return AbortNode(state, "Failed to write Sapling subtree metadata");
+            }
+        }
+        if (!completedOrchardSubtrees.empty()) {
+            if (!pblocktree->WriteShieldedSubtrees(ORCHARDFRONTIER, completedOrchardSubtrees)) {
+                return AbortNode(state, "Failed to write Orchard subtree metadata");
+            }
+        }
+    }
+
     // add this block to the view's block chain
     view.SetBestBlock(pindex->GetBlockHash());
 
@@ -4644,6 +4769,13 @@ bool static DisconnectTip(CValidationState &state, bool fBare = false) {
     // Update chainActive and related variables.
     UpdateTip(pindexDelete->pprev);
 
+    if (!pblocktree->TruncateShieldedSubtrees(SAPLINGFRONTIER, chainActive.Height())) {
+        return AbortNode(state, "Failed to truncate Sapling subtree metadata");
+    }
+    if (!pblocktree->TruncateShieldedSubtrees(ORCHARDFRONTIER, chainActive.Height())) {
+        return AbortNode(state, "Failed to truncate Orchard subtree metadata");
+    }
+
     // Get the current commitment tree
     SproutMerkleTree newSproutTree;
     SaplingMerkleTree newSaplingTree;
@@ -7029,6 +7161,14 @@ bool static LoadBlockIndexDB()
     LOCK(cs_main);
     chainActive.SetTip(it->second);
 
+    {
+        std::string subtreeError;
+        if (!AlignPersistedShieldedSubtrees(subtreeError)) {
+            LogPrintf("Failed to align persisted shielded subtree metadata: %s\n", subtreeError);
+            return false;
+        }
+    }
+
     // Set hashFinalSproutRoot for the end of best chain
     it->second->hashFinalSproutRoot = pcoinsTip->GetBestAnchor(SPROUT);
 

src/main.h

@@ -916,6 +916,7 @@ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHea
 bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos,bool checkPOW);
 bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex,bool checkPOW);
 bool PruneOneBlockFile(bool tempfile, const int fileNumber);
+bool AlignPersistedShieldedSubtrees(std::string& error);
 
 /** Functions for validating blocks and updating the block tree */
 

src/rpc/blockchain.cpp

@@ -46,17 +46,74 @@
 #include "komodo_kv.h"
 #include "komodo_gateway.h"
 #include "rpc/rawtransaction.h"
+#include "txdb.h"
 
 #include <stdint.h>
 
 #include <univalue.h>
 
+#include <algorithm>
+#include <cctype>
+#include <limits>
 #include <regex>
+#include <string>
 
 #include "cc/CCinclude.h"
 
 using namespace std;
 
+namespace {
+
+enum class ShieldedSubtreeProtocol {
+    Sapling,
+    Orchard,
+};
+
+ShieldedSubtreeProtocol ParseShieldedSubtreeProtocol(const UniValue& value)
+{
+    std::string protocol = value.get_str();
+    std::transform(protocol.begin(), protocol.end(), protocol.begin(), [](unsigned char c) {
+        return std::tolower(c);
+    });
+
+    if (protocol == "sapling") {
+        return ShieldedSubtreeProtocol::Sapling;
+    }
+    if (protocol == "orchard") {
+        return ShieldedSubtreeProtocol::Orchard;
+    }
+
+    throw JSONRPCError(RPC_INVALID_PARAMETER, "shielded protocol must be \"sapling\" or \"orchard\"");
+}
+
+uint64_t ParseNonNegativeUint64(const UniValue& value, const std::string& fieldName)
+{
+    if (!(value.isNum() || value.isStr())) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, fieldName + " must be a non-negative integer");
+    }
+
+    int64_t parsed = value.get_int64();
+    if (parsed < 0) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, fieldName + " must be a non-negative integer");
+    }
+
+    return static_cast<uint64_t>(parsed);
+}
+
+ShieldedType ToShieldedType(ShieldedSubtreeProtocol protocol)
+{
+    switch (protocol) {
+    case ShieldedSubtreeProtocol::Sapling:
+        return SAPLINGFRONTIER;
+    case ShieldedSubtreeProtocol::Orchard:
+        return ORCHARDFRONTIER;
+    }
+
+    throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown shielded protocol");
+}
+
+} // namespace
+
 // TODO: remove
 //extern int32_t KOMODO_INSYNC;
 //extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry);
@@ -2335,6 +2392,87 @@ UniValue z_gettreestatelegacy(const UniValue& params, bool fHelp, const CPubKey&
     return res;
 }
 
+UniValue z_getsubtreesbyindex(const UniValue& params, bool fHelp, const CPubKey& mypk)
+{
+    if (fHelp || params.size() < 2 || params.size() > 3)
+        throw runtime_error(
+            "z_getsubtreesbyindex \"sapling|orchard\" start_index ( limit )\n"
+            "Return roots of completed shielded 2^16 subtrees from the active chain.\n"
+            "\nArguments:\n"
+            "1. \"sapling|orchard\"      (string, required) Shielded protocol to query\n"
+            "2. start_index              (numeric, required) Subtree index to start from\n"
+            "3. limit                    (numeric, optional, default=0) Maximum number of subtrees to return, or 0 for all\n"
+            "\nResult:\n"
+            "[\n"
+            "  {\n"
+            "    \"index\": n,                    (numeric) subtree index\n"
+            "    \"root\": \"hex\",               (string) subtree root hash\n"
+            "    \"completingBlockHash\": \"hex\",(string) block hash that completed the subtree\n"
+            "    \"completingBlockHeight\": n     (numeric) block height that completed the subtree\n"
+            "  }\n"
+            "]\n"
+            "\nExamples:\n"
+            + HelpExampleCli("z_getsubtreesbyindex", "\"sapling\" 0 10")
+            + HelpExampleRpc("z_getsubtreesbyindex", "\"sapling\", 0, 10")
+        );
+
+    ShieldedSubtreeProtocol protocol = ParseShieldedSubtreeProtocol(params[0]);
+    uint64_t startIndex = ParseNonNegativeUint64(params[1], "start_index");
+    uint64_t limit = params.size() == 3 ? ParseNonNegativeUint64(params[2], "limit") : 0;
+    if (limit > std::numeric_limits<uint32_t>::max()) {
+        throw JSONRPCError(RPC_INVALID_PARAMETER, "limit exceeds uint32 range");
+    }
+
+    LOCK(cs_main);
+
+    std::string subtreeError;
+    if (!AlignPersistedShieldedSubtrees(subtreeError)) {
+        throw JSONRPCError(RPC_DATABASE_ERROR, subtreeError);
+    }
+
+    std::vector<std::pair<uint64_t, ShieldedSubtreeData>> subtrees;
+    if (!pblocktree->ReadShieldedSubtrees(ToShieldedType(protocol), startIndex, static_cast<uint32_t>(limit), subtrees)) {
+        throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read persisted subtree metadata");
+    }
+    UniValue result(UniValue::VARR);
+
+    for (const auto& it : subtrees) {
+        const uint64_t subtreeIndex = it.first;
+        const ShieldedSubtreeData& subtree = it.second;
+        if (subtree.nHeight < 0) {
+            throw JSONRPCError(
+                RPC_DATABASE_ERROR,
+                strprintf(
+                    "subtree index %llu has invalid completion height %d",
+                    static_cast<unsigned long long>(subtreeIndex),
+                    subtree.nHeight));
+        }
+        const CBlockIndex* completingBlock = chainActive[subtree.nHeight];
+        if (completingBlock == nullptr) {
+            throw JSONRPCError(
+                RPC_DATABASE_ERROR,
+                strprintf("missing active block index at subtree completion height %d", subtree.nHeight));
+        }
+        if (completingBlock->GetBlockHash() != subtree.blockHash) {
+            throw JSONRPCError(
+                RPC_DATABASE_ERROR,
+                strprintf(
+                    "persisted subtree metadata mismatch at index %llu and height %d",
+                    static_cast<unsigned long long>(subtreeIndex),
+                    subtree.nHeight));
+        }
+
+        UniValue entry(UniValue::VOBJ);
+        entry.push_back(Pair("index", static_cast<int64_t>(subtreeIndex)));
+        entry.push_back(Pair("root", HexStr(subtree.root.begin(), subtree.root.end())));
+        entry.push_back(Pair("completingBlockHash", subtree.blockHash.GetHex()));
+        entry.push_back(Pair("completingBlockHeight", static_cast<int64_t>(subtree.nHeight)));
+        result.push_back(entry);
+    }
+
+    return result;
+}
+
 /**
  * @brief Convert memory pool information to JSON format
  * @return JSON object containing memory pool statistics
@@ -2576,6 +2714,7 @@ static const CRPCCommand commands[] =
     { "blockchain",         "getchaintips",           &getchaintips,           true  },
     { "blockchain",         "z_gettreestate",         &z_gettreestate,         true  },
     { "blockchain",         "z_gettreestatelegacy",   &z_gettreestatelegacy,   true  },
+    { "blockchain",         "z_getsubtreesbyindex",   &z_getsubtreesbyindex,   true  },
     { "blockchain",         "getchaintxstats",        &getchaintxstats,        true  },
     { "blockchain",         "getdifficulty",          &getdifficulty,          true  },
     { "blockchain",         "getmempoolinfo",         &getmempoolinfo,         true  },

src/rpc/server.h

@@ -437,6 +437,7 @@ extern UniValue verifychain(const UniValue& params, bool fHelp, const CPubKey& m
 extern UniValue getchaintips(const UniValue& params, bool fHelp, const CPubKey& mypk);
 extern UniValue z_gettreestate(const UniValue& params, bool fHelp, const CPubKey& mypk);
 extern UniValue z_gettreestatelegacy(const UniValue& params, bool fHelp, const CPubKey& mypk);
+extern UniValue z_getsubtreesbyindex(const UniValue& params, bool fHelp, const CPubKey& mypk);
 extern UniValue invalidateblock(const UniValue& params, bool fHelp, const CPubKey& mypk);
 extern UniValue reconsiderblock(const UniValue& params, bool fHelp, const CPubKey& mypk);
 extern UniValue getspentinfo(const UniValue& params, bool fHelp, const CPubKey& mypk);