From 516c41fff32b3345aa34a42f9fd00f4fd0c2db3b Mon Sep 17 00:00:00 2001 From: Mark Friedenbach Date: Thu, 21 Jan 2016 16:27:01 -0800 Subject: [PATCH 1/4] Allow arbitrary commitment tree structure. The witness root is allowed to be placed at an arbitrary position up to seven layers deep in a Merkle tree structure. The witness nonce is now the branch through the commitment tree to the witness root, and a single byte is added to the commitment output specifying this path in compact form. This allows other consensus commitments to be added in the future with a minimal number of bytes and without requiring a certain position for each commitment in the tree. --- src/main.cpp | 72 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 21eefe6f4479..b12a78c1eb9e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3353,19 +3353,35 @@ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& pa return (VersionBitsState(pindexPrev, params, Consensus::DEPLOYMENT_WITNESS, versionbitscache) == THRESHOLD_ACTIVE); } +inline bool IsCoinbaseCommitment(const CTxOut& txout) +{ + if ( txout.scriptPubKey.size() >= 39 + && txout.scriptPubKey[0] == OP_RETURN + && txout.scriptPubKey[1] >= 0x24 + && txout.scriptPubKey[1] <= 0x4b + && txout.scriptPubKey[2] == 0xaa + && txout.scriptPubKey[3] == 0x21 + && txout.scriptPubKey[4] == 0xa9 + && txout.scriptPubKey[5] == 0xed) + { + return true; + } + + return false; +} + void UpdateUncommitedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) { int commitpos = -1; for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { - if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + if (IsCoinbaseCommitment(block.vtx[0].vout[o])) { commitpos = o; } } - static const std::vector nonce(32, 0x00); if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0].wit.IsEmpty()) { block.vtx[0].wit.vtxinwit.resize(1); block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.resize(1); - block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = nonce; + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = std::vector(1, 0x00); } } @@ -3374,7 +3390,7 @@ std::vector GenerateCoinbaseCommitment(CBlock& block, const CBloc std::vector commitment; int commitpos = -1; for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { - if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + if (IsCoinbaseCommitment(block.vtx[0].vout[o])) { commitpos = o; } } @@ -3385,21 +3401,20 @@ std::vector GenerateCoinbaseCommitment(CBlock& block, const CBloc break; } } - std::vector ret(32, 0x00); if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) { if (commitpos == -1) { uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); - CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); CTxOut out; out.nValue = 0; - out.scriptPubKey.resize(38); + out.scriptPubKey.resize(39); out.scriptPubKey[0] = OP_RETURN; out.scriptPubKey[1] = 0x24; out.scriptPubKey[2] = 0xaa; out.scriptPubKey[3] = 0x21; out.scriptPubKey[4] = 0xa9; out.scriptPubKey[5] = 0xed; - memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); + out.scriptPubKey[6] = 0x01; + memcpy(&out.scriptPubKey[7], witnessroot.begin(), 32); commitment = std::vector(out.scriptPubKey.begin(), out.scriptPubKey.end()); const_cast*>(&block.vtx[0].vout)->push_back(out); block.vtx[0].UpdateHash(); @@ -3465,16 +3480,28 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn // Validation for witness commitments. // * We compute the witness hash (which is the hash including witnesses) of all the block's transactions, except the // coinbase (where 0x0000....0000 is used instead). - // * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness nonce (unconstrained). + // * The coinbase scriptWitness is a stack of a serialize N*32-byte vector, where N is the number of hashes in the + // (unconstrained) Merkle path from the commitment to the witness tree (0 <= N <= 7). // * We build a merkle tree with all those witness hashes as leaves (similar to the hashMerkleRoot in the block header). - // * The must be at least one output whose scriptPubKey is a single 36-byte push, the first 4 bytes of which are - // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256(witness root, witness nonce). In case there are - // multiple, the last one is used. + // * There must be at least one output whose scriptPubKey is a single >=37-byte push, the first 4 bytes of which are + // {0xaa, 0x21, 0xa9, 0xed}, followed by between 0 and 38 unconstrained bytes, then a single byte encoding of the + // path from the commitment root to the witness root, and the final 32 bytes are the commitment root. In case there + // are multiple, the last one is used. bool fHaveWitness = false; if (IsWitnessEnabled(pindexPrev, consensusParams)) { int commitpos = -1; for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { - if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + if ( block.vtx[0].vout[o].scriptPubKey.size() >= 39 + && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN + && block.vtx[0].vout[o].scriptPubKey[1] >= 0x24 + && block.vtx[0].vout[o].scriptPubKey[1] <= 0x4b + && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa + && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 + && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 + && block.vtx[0].vout[o].scriptPubKey[5] == 0xed + && block.vtx[0].vout[o].scriptPubKey[ + block.vtx[0].vout[o].scriptPubKey.size()-33] != 0x00) + { commitpos = o; } } @@ -3484,11 +3511,24 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn // The malleation check is ignored; as the transaction tree itself // already does not permit it, it is impossible to trigger in the // witness tree. - if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { + const CScript& commitscript = block.vtx[0].vout[commitpos].scriptPubKey; + const unsigned char& witnesspath = commitscript[commitscript.size()-33]; + size_t witnessdepth = 0; + for (int pos = 0; pos < 8; ++pos) + if (witnesspath & (1< branch; + ds >> branch; + hashWitness = ComputeMerkleRootFromBranch(hashWitness, branch, witnesspath ^ (1< Date: Fri, 22 Jan 2016 08:19:17 -0800 Subject: [PATCH 2/4] Add mistate output to CSHA256. This allows the intermediate state of a SHA-256 run to be saved for future resumption, or in the case of fast Merkle trees to perform a non-padded hash. --- src/crypto/sha256.cpp | 11 +++++++++++ src/crypto/sha256.h | 1 + 2 files changed, 12 insertions(+) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 5b9f00a0a2dc..f76193daee3c 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -171,6 +171,11 @@ void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE]) WriteBE64(sizedesc, bytes << 3); Write(pad, 1 + ((119 - (bytes % 64)) % 64)); Write(sizedesc, 8); + Midstate(hash, NULL, NULL); +} + +void CSHA256::Midstate(unsigned char hash[OUTPUT_SIZE], uint64_t* len, unsigned char *buffer) +{ WriteBE32(hash, s[0]); WriteBE32(hash + 4, s[1]); WriteBE32(hash + 8, s[2]); @@ -179,6 +184,12 @@ void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE]) WriteBE32(hash + 20, s[5]); WriteBE32(hash + 24, s[6]); WriteBE32(hash + 28, s[7]); + if (len) { + *len = bytes << 3; + } + if (buffer) { + memcpy(buffer, buf, bytes % 64); + } } CSHA256& CSHA256::Reset() diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index 5b15b6a233b1..1123b8a6f127 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -22,6 +22,7 @@ class CSHA256 CSHA256(); CSHA256& Write(const unsigned char* data, size_t len); void Finalize(unsigned char hash[OUTPUT_SIZE]); + void Midstate(unsigned char hash[OUTPUT_SIZE], uint64_t* bytes, unsigned char *buffer); CSHA256& Reset(); }; From 6dadca649d72af699855066c1d78326765dd8397 Mon Sep 17 00:00:00 2001 From: Mark Friedenbach Date: Fri, 22 Jan 2016 08:50:33 -0800 Subject: [PATCH 3/4] Add fast Merkle branch functions. A fast Merkle branch uses midstate to perform a single SHA-256 compression per branch, and is not vulnerable to CVE-2012-2459. It produces different hashes though, so can only be used for new hash trees going forward. --- src/consensus/merkle.cpp | 72 ++++++++++++++++++++++++++++++++++++---- src/consensus/merkle.h | 10 ++++++ 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 35f7d2e05a93..b70dc1d38257 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -42,14 +42,32 @@ root. */ +typedef enum { + MERKLE_COMPUTATION_MUTABLE = 0x1, + MERKLE_COMPUTATION_FAST = 0x2 +} merklecomputationopts; + +static void MerkleHash_Hash256(uint256& parent, const uint256& left, const uint256& right) { + CHash256().Write(left.begin(), 32).Write(right.begin(), 32).Finalize(parent.begin()); +} + +static void MerkleHash_Sha256Midstate(uint256& parent, const uint256& left, const uint256& right) { + CSHA256().Write(left.begin(), 32).Write(right.begin(), 32).Midstate(parent.begin(), NULL, NULL); +} + /* This implements a constant-space merkle root/path calculator, limited to 2^32 leaves. */ -static void MerkleComputation(const std::vector& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector* pbranch) { +static void MerkleComputation(const std::vector& leaves, uint256* proot, bool* pmutated, uint32_t branchpos, std::vector* pbranch, merklecomputationopts flags) { if (pbranch) pbranch->clear(); if (leaves.size() == 0) { if (pmutated) *pmutated = false; if (proot) *proot = uint256(); return; } + bool fMutable = flags & MERKLE_COMPUTATION_MUTABLE; + void (*MerkleHash)(uint256&, const uint256&, const uint256&) = MerkleHash_Hash256; + if (flags & MERKLE_COMPUTATION_FAST) { + MerkleHash = MerkleHash_Sha256Midstate; + } bool mutated = false; // count is the number of leaves processed so far. uint32_t count = 0; @@ -80,7 +98,7 @@ static void MerkleComputation(const std::vector& leaves, uint256* proot } } mutated |= (inner[level] == h); - CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + MerkleHash(h, inner[level], h); } // Store the resulting hash at inner position level. inner[level] = h; @@ -106,7 +124,9 @@ static void MerkleComputation(const std::vector& leaves, uint256* proot if (pbranch && matchh) { pbranch->push_back(h); } - CHash256().Write(h.begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + if (fMutable) { + MerkleHash(h, h, h); + } // Increment count to the value it would have if two entries at this // level had existed. count += (((uint32_t)1) << level); @@ -121,7 +141,7 @@ static void MerkleComputation(const std::vector& leaves, uint256* proot matchh = true; } } - CHash256().Write(inner[level].begin(), 32).Write(h.begin(), 32).Finalize(h.begin()); + MerkleHash(h, inner[level], h); level++; } } @@ -132,13 +152,13 @@ static void MerkleComputation(const std::vector& leaves, uint256* proot uint256 ComputeMerkleRoot(const std::vector& leaves, bool* mutated) { uint256 hash; - MerkleComputation(leaves, &hash, mutated, -1, NULL); + MerkleComputation(leaves, &hash, mutated, -1, NULL, MERKLE_COMPUTATION_MUTABLE); return hash; } std::vector ComputeMerkleBranch(const std::vector& leaves, uint32_t position) { std::vector ret; - MerkleComputation(leaves, NULL, NULL, position, &ret); + MerkleComputation(leaves, NULL, NULL, position, &ret, MERKLE_COMPUTATION_MUTABLE); return ret; } @@ -155,6 +175,46 @@ uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector& leaves) { + uint256 hash; + MerkleComputation(leaves, &hash, NULL, -1, NULL, MERKLE_COMPUTATION_FAST); + return hash; +} + +std::vector ComputeFastMerkleBranch(const std::vector& leaves, uint32_t position) { + std::vector ret; + MerkleComputation(leaves, NULL, NULL, position, &ret, MERKLE_COMPUTATION_FAST); + return ret; +} + +uint256 ComputeFastMerkleRootFromBranch(const uint256& leaf, const std::vector& vMerkleBranch, uint32_t nIndex) { + size_t max = 0; + for (int i = 0; i < 32; ++i) + if (nIndex & ((uint32_t)1)< vMerkleBranch.size()) { + int i; + for (i = max-1; i >= 0; --i) + if (!(nIndex & ((uint32_t)1)<>1) | + ((((uint32_t)1)<< i )-1); + --max; + } + uint256 hash = leaf; + for (std::vector::const_iterator it = vMerkleBranch.begin(); it != vMerkleBranch.end(); ++it) { + if (nIndex & 1) { + MerkleHash_Sha256Midstate(hash, *it, hash); + } else { + MerkleHash_Sha256Midstate(hash, hash, *it); + } + nIndex >>= 1; + } + return hash; +} + uint256 BlockMerkleRoot(const CBlock& block, bool* mutated) { std::vector leaves; diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 194aea9b75dc..576cba9b7497 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -16,6 +16,16 @@ uint256 ComputeMerkleRoot(const std::vector& leaves, bool* mutated = NU std::vector ComputeMerkleBranch(const std::vector& leaves, uint32_t position); uint256 ComputeMerkleRootFromBranch(const uint256& leaf, const std::vector& branch, uint32_t position); +/* + * Has similar API semantics, but produces Merkle roots and validates + * branches 3x as fast, and without the mutation vulnerability. Cannot + * be substituted for the non-fast variants because the hash values are + * different. + */ +uint256 ComputeFastMerkleRoot(const std::vector& leaves); +std::vector ComputeFastMerkleBranch(const std::vector& leaves, uint32_t position); +uint256 ComputeFastMerkleRootFromBranch(const uint256& leaf, const std::vector& branch, uint32_t position); + /* * Compute the Merkle root of the transactions in a block. * *mutated is set to true if a duplicated subtree was found. From f2ad398fcd5f13a0c700de7a6fa724ec661f1eea Mon Sep 17 00:00:00 2001 From: Mark Friedenbach Date: Fri, 22 Jan 2016 08:56:14 -0800 Subject: [PATCH 4/4] Switch to fast Merkle trees for witness. --- src/consensus/merkle.cpp | 4 ++-- src/consensus/merkle.h | 2 +- src/main.cpp | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index b70dc1d38257..989c5b193f0f 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -225,7 +225,7 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated) return ComputeMerkleRoot(leaves, mutated); } -uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated) +uint256 BlockWitnessMerkleRoot(const CBlock& block) { std::vector leaves; leaves.resize(block.vtx.size()); @@ -233,7 +233,7 @@ uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated) for (size_t s = 1; s < block.vtx.size(); s++) { leaves[s] = block.vtx[s].GetWitnessHash(); } - return ComputeMerkleRoot(leaves, mutated); + return ComputeFastMerkleRoot(leaves); } std::vector BlockMerkleBranch(const CBlock& block, uint32_t position) diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 576cba9b7497..71eb27ad196a 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -36,7 +36,7 @@ uint256 BlockMerkleRoot(const CBlock& block, bool* mutated = NULL); * Compute the Merkle root of the witness transactions in a block. * *mutated is set to true if a duplicated subtree was found. */ -uint256 BlockWitnessMerkleRoot(const CBlock& block, bool* mutated = NULL); +uint256 BlockWitnessMerkleRoot(const CBlock& block); /* * Compute the Merkle branch for the tree of transactions in a block, for a diff --git a/src/main.cpp b/src/main.cpp index b12a78c1eb9e..c357af2a3537 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3403,7 +3403,7 @@ std::vector GenerateCoinbaseCommitment(CBlock& block, const CBloc } if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) { if (commitpos == -1) { - uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); + uint256 witnessroot = BlockWitnessMerkleRoot(block); CTxOut out; out.nValue = 0; out.scriptPubKey.resize(39); @@ -3506,8 +3506,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn } } if (commitpos != -1) { - bool malleated = false; - uint256 hashWitness = BlockWitnessMerkleRoot(block, &malleated); + uint256 hashWitness = BlockWitnessMerkleRoot(block); // The malleation check is ignored; as the transaction tree itself // already does not permit it, it is impossible to trigger in the // witness tree.