From 1da936a59b87bb7de8867ebee8d8f47cb269b026 Mon Sep 17 00:00:00 2001 From: AmiT177 Date: Fri, 19 Jan 2018 10:49:48 +0200 Subject: [PATCH 01/79] Added HOdlcoin support. (#350) * Added HOdlcoin support. * Update coins.py * HOdlcoin block test --- lib/coins.py | 15 +++++++++++++++ tests/blocks/hodlcoin_mainnet_38200.json | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/blocks/hodlcoin_mainnet_38200.json diff --git a/lib/coins.py b/lib/coins.py index 5b5bf6d1e..3d6ad548c 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -380,6 +380,21 @@ class BitcoinMixin(object): '4ff763ae46a2a6c172b3f1b60a8ce26f') RPC_PORT = 8332 +class HOdlcoin(Coin): + NAME = "HOdlcoin" + SHORTNAME = "HODLC" + NET = "mainnet" + BASIC_HEADER_SIZE = 88 + P2PKH_VERBYTE = bytes.fromhex("28") + P2SH_VERBYTES = [bytes.fromhex("05")] + WIF_BYTE = bytes.fromhex("a8") + GENESIS_HASH = ('008872e5582924544e5c707ee4b839bb' + '82c28a9e94e917c94b40538d5658c04b') + DESERIALIZER = lib_tx.DeserializerSegWit + TX_COUNT = 258858 + TX_COUNT_HEIGHT = 382138 + TX_PER_BLOCK = 5 + class BitcoinCash(BitcoinMixin, Coin): NAME = "BitcoinCash" diff --git a/tests/blocks/hodlcoin_mainnet_38200.json b/tests/blocks/hodlcoin_mainnet_38200.json new file mode 100644 index 000000000..dac681111 --- /dev/null +++ b/tests/blocks/hodlcoin_mainnet_38200.json @@ -0,0 +1,20 @@ +{ + "hash" : "0000001549849c02a62b68f5cb7fb7beb1bbcfe05a031b3c64e5b7c0d047b47d", + "size" : 2885, + "height" : 38200, + "merkleroot" : "7caa3260697d878875c5535e4fa0b16f1957bb472bff796607d0fa1a991a52c6", + "tx" : [ + "b32e0efdd5978e6da635b97edf80b1a268d8c1401e3137a00296e5c096b73aad", + "e0b110989d33b04925fede9ce61a58e416adf90288e4e6e07409bd754090a6ab", + "65400cd15f438cd0c5bc805951ea48c1b52d88bf4ba1746b359d86e510261d55", + "850dfd53bb95759f49c9ea7188295f5581ecf53a6e1c551e412018865f62f325", + "875f4291642805ca8f6d0871fe0a20a42a6300c5f1c610d81f3ddc06d1b71249", + "313ef761f0312cf5a221069cb293c62ea16431191d6c4e69b2eeaee3c6afe114", + "52e0a1024321f6cfee001419333aa5e729bef13fd05c9736ead3aa80b2264652" + ], + "time" : 1460515581, + "nonce" : 4216, + "bits" : "1d6c0663", + "previousblockhash" : "0000005135889a6a06fe0239ca5976a3812f888199573994c0b7ae2e51e5b275", + "block" : "0400000075b2e5512eaeb7c09439579981882f81a37659ca3902fe066a9a883551000000c6521a991afad0076679ff2b47bb57196fb1a04f5e53c57588877d696032aa7cfdb20d5763066c1d78100000d78501005ba1c1120701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200338950004fdb20d57082ffffd28040000000d2f6e6f64655374726174756d2f00000000016226062a010000001976a91416513c9f44366d527e2a0f123fa8ac80cbe1ef2e88ac0000000001000000083098f00d1ee573854c92e863d8be29c7968f16c5586350da067abdb8192c772d000000006a473044022057c8d253a719e9c3b3b9fcf0cc5011b16993263df8abd537d6990720488a19120220250cfb8204ed1780ab4cae29b36f7f624d4e1cdd430267282753ed908a641419012102034f8509f97c5c77ece89addd35487dcdab0b6dfae2f83717b591b1e9d150a8afeffffff66012f9cb2a1c734027665a722cbbae2280a59448efd5d4fd9d039763a00880c000000006a4730440220425f8acd95dbc671dc5497485ce02c5d0843f66259229fcc899bea20aa4bff7002207a8cd20837b3f64afb9162610766c4f61dedf1624777822f667bcd937e2959d6012102034f8509f97c5c77ece89addd35487dcdab0b6dfae2f83717b591b1e9d150a8afeffffffacce0ad69292b43689c0c3224ae13a9c2d67e2ae6d6a0a91d246156721e4e2e5010000006a473044022074fcb69c2b5299ca538e92aba61b3a05a9e3aba2673fb88086804b58ebc2e4e202203567ebbc15fa31646bc2fa26980b99539241b15d198ea0688238abd70b65583801210352d97dcc509f8b89ffe658c3e316909d59e5686a38f293d54baf912c5871bb7ffeffffff8abd190dbe8be8c07b00229edc8ed98986a1e1162bad68aae0dd2802f8353cc8000000006b4830450221009733fe224b02c346bcbbfb08b4212cba9c094d98e7b0e6dd44dd9fdbb87f028d02203c574435cfe7e61e5ef667053e0f3c23ef288c9d2ec62c61dfd9e38351fd9d4f012102034f8509f97c5c77ece89addd35487dcdab0b6dfae2f83717b591b1e9d150a8afeffffff705d27362b41d96aaa1d610f5b4066bf96b17d90f982777192df2c032b50a23c010000006a4730440220344c81c778061334da6c93eef93ce43f2b0c362a10890f80ed34bec1310f1c0002203df92418d4c14dcd243199a59f24f127977fcd80ee8a19f14b52c2c784d8a1ff012102ca71adadfd79458ce109e24dd602ab9c4bb23bdc836e5e5e0312be38f87d66c3feffffff1f141b00c7a090a87b9a7b991668434951175e50b9fb4642cb7af48d4d918308000000006a47304402206d319617f0623b9e46d6a125dcb7e17296ae267cd3ad032879054203aad99cbc0220181d84238e422cc3cc708c8f26b9c874b4395d193388dc697cea7ee72b3d8cfe012102034f8509f97c5c77ece89addd35487dcdab0b6dfae2f83717b591b1e9d150a8afeffffffd5aeb1514f7ad8ad3ba0d41096354d3e574f111a17785fe0c370a55ce69902d2010000006b483045022100ab92696a0deaaed9ccc8bfef4efff9e94f276a34ce3f9436abb5c4e1f5c8f40902206403f8f1c15d7a54ea1f19b10fb016cc9f76a93459a6c75f09726fd1cee2148e012103cdea19048dae1411d73e341bc854db981c7c32000bc356ccaaf2e55939b6fa22feffffff3e20ab1501ba7a1ebc060acf8ebb8c6fd88fb4120fbc1c87930b339f7d8bbedc000000006b483045022100fb1b657494c26082fb405ee1690c4a95134de5731f409f3bbf746e7ebffa38ca02203747e51315fad172b4270653c7f523d4e3fa22d12c394fae36f57031924a1b7e012102837aa29741e354ce193a89974f720be1cfd8ec05308752360d45f4dcbf7cdcaffeffffff029ed3cede050000001976a914583fc2f568444d2d25bfbc3eff4bb30417975f1f88acdaad0f00000000001976a914c5d9c029f13a6e52f5707e5d5c256512ab2d362a88ac379500000100000001c9001545d3ec51a844a9fe71393e6130097750474d1c5ea0bab35eb94847ad10000000006a4730440220091e9a048274d8cb424e27caa932ddb2538eda7c0c41771cea7b689d937fc902022033b32f8c897af50751fb9851ee2b1a1487b49be38a3b3cd5657a0bab5739aab6012102034f8509f97c5c77ece89addd35487dcdab0b6dfae2f83717b591b1e9d150a8afeffffff02651e2400000000001976a914f973d158b7e344ddc489aca3816ed683248b71f888acf295142a010000001976a914dd97511cd56fadf884c140db41f6587bfd9c970788ac379500000100000001a3f901d7e993aad2f310be88d89e9edad5df1309e3803e372ee9f192f5d212fc010000006b483045022100c0aba433782ce6b2cfe0aec70e3b6f5752121426af75f4ea78f2547d7773312b022011ce21a7777a05ac2495a6859a744af8e7dfa78d604dab581f91af5636bd6dd40121030d4908889b01ed25478f46b8975fe618fb52f8156ebbf996119ea28423becd11feffffff022c00de7c000000001976a914fca90b447f4a4a84c5a36b115fc0da953ec9535a88ac0562873d000000001976a914eeaea63253ce993b246fd07caa73da3e41de307088ac37950000010000000139b2b675d3937e03da186560f1cb0188fadf4b7784f4502f2ebb822c24995d12000000006a4730440220764af6dfc020af5fa95ab2734f97bac4a197d0e4cad1b6647a3b2c7f7c7d047002206e0cbc26b3df33d39767c7e82a4ac7a75a038bae23f0ab92f02f6facae239d360121036acf3bb480a8b3db917cb7eb8a8efa1af18ec4bc89a06dabfc9ef0ecc35fdbaffeffffff029042f816000000001976a914c76c36027bc5fab4a4c1f3572a9d85f0372efa5d88ace169f61d000000001976a914c61c24e63d8686447ddad1a1c0f0c30f53d5e05388ac3795000001000000012a8c24b40a8e40027a9cd301566e89c55816de80118150b6289d34e247dfbf1b000000006a47304402205a6a3862747a3746d5bd0376fe5ea35158d582086747a928745613b1170f862f02207598a20f9cc6f3c21188449fb9df4e62929182b8dca168a4d111ea1f1943c488012102edee8123a5ff10ff99363104aa81de6f17905ae1d901c592f80c55395eea638afeffffff02339e8009000000001976a91420e212cd244a96b879aced1560f3e574a2e382e188ac8171532d000000001976a914651608f0934e6fe284dd2348fa5f45f4a30ea52588ac379500000100000003737798484255c4b51307caddacd510f627c4b13fd936d33facc6524a087cff08010000006b483045022100c84c0373a28ce3c1752f4982a232212bb7a4a80d3385acae24cea8cef1c7e687022063863c4e737b0cf4c73ed4b6677daadcca76721d34ba17f1025f56e08cce476c0121036fa9f8511381af76f8a097b85fddc80f489ed32e0c12b9dd9fdd3f9a0b0dbc49feffffff8b60f2008465f6d8bb5dd8d21dd786bc516e6e4b6f45ac59e484c1d9a29e4734000000006a47304402202d8ac18ee89547367576ce68a21f22fea595c69988de7952f2985dbefe3ae87102204a4ce511e56119392f20ed3be448c293160a7968561174231adfe2fac2670900012102da50a27feae936d69ee4a4f26a28c9301cc3374722fbd222cc595f9c561772e1feffffff780b4dfa2695837323b7ec5f1a75de634270b575bab72245029b84be2b8928d4000000006b483045022100f61cebd38a898a56be862b15d27e20af438717e13cce8a0fe5f6b855f2ea3d520220453ad08f503518e1d12d6c9b0ee5fef5f4c6fd2170e7a5750945c4c6599bda39012103909ef210f4892548c4dc07c5cb41c493000a6a47ad7d94b69ac8b537208ee913feffffff0231695200000000001976a9141cd6aabfe6b4ce339d9d6a2d3464a68e3bbf610488ace9f60406000000001976a914a3c0b7a0a60ee0fde80307ee6d0bc58dc5cc611788ac37950000" +} From 2d7403f2efed7e8f33c5cb93e2cd9144415cbb9f Mon Sep 17 00:00:00 2001 From: ThomasV Date: Sat, 20 Jan 2018 13:51:38 +0100 Subject: [PATCH 02/79] New protocol: (#330) - add method mempool.get_fee_histogram - bump protocol version to 1.2 --- docs/PROTOCOL.rst | 14 +++++++++ server/controller.py | 8 ++++++ server/mempool.py | 68 +++++++++++++++++++++++++++++++++++--------- server/session.py | 6 ++++ server/version.py | 2 +- 5 files changed, 84 insertions(+), 14 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 6009d3d3e..c30d615a0 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -774,6 +774,20 @@ Subscribe to a script hash. [**scripthash**, **status**] +mempool.get_fee_histogram +========================= + +Return a histogram of the fee rates paid by transactions in the memory +pool, weighted by transaction size. + +The histogram is an array of (fee, vsize) values, where vsize_n is the +cumulative virtual size of mempool transactions with a fee rate in the +interval [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. + +Fee intervals may have variable size. The choice of appropriate +intervals is currently not part of the protocol. + + server.add_peer =============== diff --git a/server/controller.py b/server/controller.py index e7e45b9b7..57b80c74b 100644 --- a/server/controller.py +++ b/server/controller.py @@ -825,6 +825,14 @@ async def estimatefee(self, number): number = self.non_negative_integer(number) return await self.daemon_request('estimatefee', [number]) + def mempool_get_fee_histogram(self): + '''Memory pool fee histogram. + + TODO: The server should detect and discount transactions that + never get mined when they should. + ''' + return self.mempool.get_fee_histogram() + async def relayfee(self): '''The minimum fee a low-priority tx must pay in order to be accepted to the daemon's memory pool.''' diff --git a/server/mempool.py b/server/mempool.py index 4193c831d..286539ce9 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -25,7 +25,7 @@ class MemPool(util.LoggedClass): To that end we maintain the following maps: - tx_hash -> (txin_pairs, txout_pairs) + tx_hash -> (txin_pairs, txout_pairs, tx_fee, tx_size) hashX -> set of all tx hashes in which the hashX appears A pair is a (hashX, value) tuple. tx hashes are hex strings. @@ -42,6 +42,9 @@ def __init__(self, bp, controller): self.txs = {} self.hashXs = defaultdict(set) # None can be a key self.synchronized_event = asyncio.Event() + self.fee_histogram = defaultdict(int) + self.compact_fee_histogram = [] + self.histogram_time = 0 def _resync_daemon_hashes(self, unprocessed, unfetched): '''Re-sync self.txs with the list of hashes in the daemon's mempool. @@ -52,6 +55,7 @@ def _resync_daemon_hashes(self, unprocessed, unfetched): txs = self.txs hashXs = self.hashXs touched = self.touched + fee_hist = self.fee_histogram hashes = self.daemon.cached_mempool_hashes() gone = set(txs).difference(hashes) @@ -60,7 +64,11 @@ def _resync_daemon_hashes(self, unprocessed, unfetched): unprocessed.pop(hex_hash, None) item = txs.pop(hex_hash) if item: - txin_pairs, txout_pairs = item + txin_pairs, txout_pairs, tx_fee, tx_size = item + fee_rate = tx_fee // tx_size + fee_hist[fee_rate] -= tx_size + if fee_hist[fee_rate] == 0: + fee_hist.pop(fee_rate) tx_hashXs = set(hashX for hashX, value in txin_pairs) tx_hashXs.update(hashX for hashX, value in txout_pairs) for hashX in tx_hashXs: @@ -138,6 +146,7 @@ async def main_loop(self): def _async_process_some(self, limit): pending = [] txs = self.txs + fee_hist = self.fee_histogram async def process(unprocessed): nonlocal pending @@ -160,10 +169,13 @@ async def process(unprocessed): pending.extend(deferred) hashXs = self.hashXs touched = self.touched - for hex_hash, in_out_pairs in result.items(): + for hex_hash, item in result.items(): if hex_hash in txs: - txs[hex_hash] = in_out_pairs - for hashX, value in itertools.chain(*in_out_pairs): + txs[hex_hash] = item + txin_pairs, txout_pairs, tx_fee, tx_size = item + fee_rate = tx_fee // tx_size + fee_hist[fee_rate] += tx_size + for hashX, value in itertools.chain(txin_pairs, txout_pairs): touched.add(hashX) hashXs[hashX].add(hex_hash) @@ -209,7 +221,7 @@ def process_raw_txs(self, raw_tx_map, pending): for tx_hash, raw_tx in raw_tx_map.items(): if tx_hash not in txs: continue - tx = deserializer(raw_tx).read_tx() + tx, tx_size = deserializer(raw_tx).read_tx_and_vsize() # Convert the tx outputs into (hashX, value) pairs txout_pairs = [(script_hashX(txout.pk_script), txout.value) @@ -219,7 +231,7 @@ def process_raw_txs(self, raw_tx_map, pending): txin_pairs = [(hash_to_str(txin.prev_hash), txin.prev_idx) for txin in tx.inputs] - pending.append((tx_hash, txin_pairs, txout_pairs)) + pending.append((tx_hash, txin_pairs, txout_pairs, tx_size)) # Now process what we can result = {} @@ -229,7 +241,7 @@ def process_raw_txs(self, raw_tx_map, pending): if self.stop: break - tx_hash, old_txin_pairs, txout_pairs = item + tx_hash, old_txin_pairs, txout_pairs, tx_size = item if tx_hash not in txs: continue @@ -259,7 +271,10 @@ def process_raw_txs(self, raw_tx_map, pending): if mempool_missing: deferred.append(item) else: - result[tx_hash] = (txin_pairs, txout_pairs) + # Compute fee + tx_fee = (sum(v for hashX, v in txin_pairs) - + sum(v for hashX, v in txout_pairs)) + result[tx_hash] = (txin_pairs, txout_pairs, tx_fee, tx_size) return result, deferred @@ -290,9 +305,7 @@ async def transactions(self, hashX): item = self.txs.get(hex_hash) if not item or not raw_tx: continue - txin_pairs, txout_pairs = item - tx_fee = (sum(v for hashX, v in txin_pairs) - - sum(v for hashX, v in txout_pairs)) + txin_pairs, txout_pairs, tx_fee, tx_size = item tx = deserializer(raw_tx).read_tx() unconfirmed = any(hash_to_str(txin.prev_hash) in self.txs for txin in tx.inputs) @@ -325,7 +338,36 @@ def value(self, hashX): # hashXs is a defaultdict if hashX in self.hashXs: for hex_hash in self.hashXs[hashX]: - txin_pairs, txout_pairs = self.txs[hex_hash] + txin_pairs, txout_pairs, tx_fee, tx_size = self.txs[hex_hash] value -= sum(v for h168, v in txin_pairs if h168 == hashX) value += sum(v for h168, v in txout_pairs if h168 == hashX) return value + + def get_fee_histogram(self): + now = time.time() + if now > self.histogram_time + 30: + self.update_compact_histogram() + self.histogram_time = now + return self.compact_fee_histogram + + def update_compact_histogram(self): + # For efficiency, get_fees returns a compact histogram with + # variable bin size. The compact histogram is an array of + # (fee, vsize) values. vsize_n is the cumulative virtual size + # of mempool transactions with a fee rate in the interval + # [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. Fee intervals + # are chosen so as to create tranches that contain at least + # 100kb of transactions + l = list(reversed(sorted(self.fee_histogram.items()))) + out = [] + size = 0 + r = 0 + binsize = 100000 + for fee, s in l: + size += s + if size + r > binsize: + out.append((fee, size)) + r += size - binsize + size = 0 + binsize *= 1.1 + self.compact_fee_histogram = out diff --git a/server/session.py b/server/session.py index 39c2727a7..91ba12ac9 100644 --- a/server/session.py +++ b/server/session.py @@ -449,6 +449,12 @@ def set_protocol_handlers(self, ptuple): 'blockchain.transaction.get': controller.transaction_get, }) + if ptuple >= (1, 2): + # New handler as of 1.2 + handlers.update({ + 'mempool.get_fee_histogram': controller.mempool_get_fee_histogram, + }) + self.electrumx_handlers = handlers def request_handler(self, method): diff --git a/server/version.py b/server/version.py index 5c69573f4..f0edc0dff 100644 --- a/server/version.py +++ b/server/version.py @@ -2,4 +2,4 @@ VERSION = 'ElectrumX 1.2.1' PROTOCOL_MIN = '0.9' -PROTOCOL_MAX = '1.1' +PROTOCOL_MAX = '1.2' From a62124468bd7a40e0bea1fe3b755d673472fa61d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 21 Jan 2018 10:28:52 -0400 Subject: [PATCH 03/79] Improve documentation --- docs/PROTOCOL.rst | 96 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index c30d615a0..a8bf59666 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -75,6 +75,58 @@ from and including the server's response to this call will use the negotiated protocol version. +Script Hashes +------------- + +A script hash is the hash of the binary bytes of the locking script +(ScriptPubKey), expressed as a hexadecimal string. The hash function +to use is given by the "hash_function" member of `server.features` +(currently "sha256" only). Like for block and transaction hashes, when +converting the big-endian binary hash to a hexadecimal string the +least-significant byte appears first, and the most-significant byte +last. + +For example, the legacy Bitcoin address from the genesis block + + 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa + +has P2PKH script + + 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac + +with SHA256 hash + + 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b + +which is sent to the server reversed as + + 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 + +By subscribing to this hash you can find P2PKH payments to that address. + +One public key for that address (the genesis block public key) is + + 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f + +which has P2PK script + + 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac + +with SHA256 hash + + 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 + +which is sent to the server reversed as + + 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 + +By subscribing to this hash you can find P2PK payments to that public +key. Note the Genesis block coinbase is unspendable and therefore not +indexed. + + Protocol Version 1.0 -------------------- @@ -774,20 +826,6 @@ Subscribe to a script hash. [**scripthash**, **status**] -mempool.get_fee_histogram -========================= - -Return a histogram of the fee rates paid by transactions in the memory -pool, weighted by transaction size. - -The histogram is an array of (fee, vsize) values, where vsize_n is the -cumulative virtual size of mempool transactions with a fee rate in the -interval [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. - -Fee intervals may have variable size. The choice of appropriate -intervals is currently not part of the protocol. - - server.add_peer =============== @@ -882,5 +920,35 @@ Get a list of features and services supported by the server. "hash_function": "sha256" } +Protocol Version 1.2 +-------------------- + +Protocol version 1.2 is the same as version `1.1` except for the +addition of a new method `mempool.get_fee_histogram`. + +All methods with taking addresses are deprecated, and will be removed +at some point in the future. You should update your code to use +`Script Hashes`_ and the scripthash methods introduced in protocol 1.1 +instead. + + +mempool.get_fee_histogram +========================= + +Request a histogram of the fee rates paid by transactions in the memory +pool, weighted by transaction size. + + mempool.get_fee_histogram() + +**Response** + + The histogram is an array of [fee, vsize] pairs, where vsize_n is + the cumulative virtual size of mempool transactions with a fee rate + in the interval [fee_(n-1), fee_n)], and fee_(n-1) > fee_n. + +Fee intervals may have variable size. The choice of appropriate +intervals is currently not part of the protocol. + + .. _JSON RPC 1.0: http://json-rpc.org/wiki/specification .. _JSON RPC 2.0: http://json-rpc.org/specification From 8891bf651f2ea319069fa6c8a15b975281b23964 Mon Sep 17 00:00:00 2001 From: xarakas Date: Thu, 25 Jan 2018 03:58:24 +0200 Subject: [PATCH 04/79] Add NYC Support (#354) --- lib/coins.py | 15 +++++++++++++++ tests/blocks/newyorkcoin_mainnet_3956926.json | 14 ++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/blocks/newyorkcoin_mainnet_3956926.json diff --git a/lib/coins.py b/lib/coins.py index 3d6ad548c..abe68b7e8 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1267,3 +1267,18 @@ class Feathercoin(Coin): PEERS = [ 'electrumx-ch-1.feathercoin.ch s t', ] + +class Newyorkcoin(AuxPowMixin, Coin): + NAME = "Newyorkcoin" + SHORTNAME = "NYC" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("3c") + P2SH_VERBYTES = [bytes.fromhex("16")] + WIF_BYTE = bytes.fromhex("bc") + GENESIS_HASH = ('5597f25c062a3038c7fd815fe46c67de' + 'dfcb3c839fbc8e01ed4044540d08fe48') + DAEMON = daemon.LegacyRPCDaemon + TX_COUNT = 5161944 + TX_COUNT_HEIGHT = 3948743 + TX_PER_BLOCK = 2 + REORG_LIMIT = 2000 diff --git a/tests/blocks/newyorkcoin_mainnet_3956926.json b/tests/blocks/newyorkcoin_mainnet_3956926.json new file mode 100644 index 000000000..268f39dde --- /dev/null +++ b/tests/blocks/newyorkcoin_mainnet_3956926.json @@ -0,0 +1,14 @@ +{ + "hash": "f08d59f8114a4e9143bcce1615f75720ccd65fd86d8c4f0e454ad41157e81a59", + "size": 248, + "height": 3956926, + "merkleroot": "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095", + "tx": [ + "0300b2fe16c049641ec63f9eb435b9d773c847bc0185c927d5b8d87cd1ad3095" + ], + "time": 1515308763, + "nonce": "1570457681", + "bits": "1b172a4e", + "previousblockhash": "10c91aefd6698f03a2820b8aa462738a5c1c21f4f975786a8eb1ba25e4b33af3", + "block": "01000000f33ab3e425bab18e6a7875f9f4211c5c8a7362a48a0b82a2038f69d6ef1ac9109530add17cd8b8d527c98501bc47c873d7b935b49e3fc61e6449c016feb20003dbc6515a4e2a171b51489b5d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff5203be603c062f503253482f04dfc6515afabe6d6dce67985c41a5f1ba5d526614e1744010b5e6a34d33c50ece5d420d75da15454e100000000000000008b8000274020000000c2f30324d515156434c59582f00000000010010a5d4e80000001976a914f0a150ec5709fae1d1814227b69cd1f0baf528c588ac00000000" +} From 12a0f00caf11504ec1b7265bab01867246a35895 Mon Sep 17 00:00:00 2001 From: Brandon Lin Date: Sat, 27 Jan 2018 13:31:49 -0800 Subject: [PATCH 05/79] simplify RSA keygen instructions (#357) --- docs/HOWTO.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index abaff5139..8bfc7f7af 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -368,10 +368,7 @@ your sign request to identify your server. They are not currently checked by the client except for the validity date. When asked for a challenge password just leave it empty and press enter:: - $ openssl genrsa -des3 -passout pass:x -out server.pass.key 2048 - $ openssl rsa -passin pass:x -in server.pass.key -out server.key - writing RSA key - $ rm server.pass.key + $ openssl genrsa -out server.key 2048 $ openssl req -new -key server.key -out server.csr ... Country Name (2 letter code) [AU]:US From dc35d3f8da397e1723e7c99a408756d3f3cf6efb Mon Sep 17 00:00:00 2001 From: WO <35449969+wo01@users.noreply.github.com> Date: Mon, 29 Jan 2018 23:47:23 +0900 Subject: [PATCH 06/79] Add Koto (#358) * add Koto * Add testfile --- lib/coins.py | 16 ++++++++++++++++ tests/blocks/koto_mainnet_60000.json | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/blocks/koto_mainnet_60000.json diff --git a/lib/coins.py b/lib/coins.py index abe68b7e8..1e426b280 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -913,6 +913,22 @@ class Zclassic(EquihashMixin, Coin): RPC_PORT = 8023 REORG_LIMIT = 800 +class Koto(Coin): + NAME = "Koto" + SHORTNAME = "KOTO" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("1836") + P2SH_VERBYTES = [bytes.fromhex("183B")] + WIF_BYTE = bytes.fromhex("80") + GENESIS_HASH = ('6d424c350729ae633275d51dc3496e16' + 'cd1b1d195c164da00f39c499a2e9959e') + DESERIALIZER = lib_tx.DeserializerZcash + TX_COUNT = 158914 + TX_COUNT_HEIGHT = 67574 + TX_PER_BLOCK = 3 + RPC_PORT = 8432 + REORG_LIMIT = 800 + class Komodo(KomodoMixin, EquihashMixin, Coin): NAME = "Komodo" SHORTNAME = "KMD" diff --git a/tests/blocks/koto_mainnet_60000.json b/tests/blocks/koto_mainnet_60000.json new file mode 100644 index 000000000..08f3935d9 --- /dev/null +++ b/tests/blocks/koto_mainnet_60000.json @@ -0,0 +1,14 @@ +{ + "hash": "09553d8cb7eabfa0a18b255d3342f171677602ace60b9be194dc562fbfa1ec32", + "size": 232, + "height": 60000, + "merkleroot": "7305c96169dc67141657d167541c50c253259d6b1150b468cbbd1a9f636e09db", + "tx": [ + "7305c96169dc67141657d167541c50c253259d6b1150b468cbbd1a9f636e09db" + ], + "time": 1516755328, + "nonce": 2806579424, + "bits": "1d0fe10c", + "previousblockhash": "66e4d540023f71e52e7e2fb68e1a2757ca6eeccb9d1b69ed5e8689d425a2b2eb", + "block": "04000000ebb2a225d489865eed691b9dcbec6eca57271a8eb62f7e2ee5713f0240d5e466db096e639f1abdcb68b450116b9d2553c2501c5467d157161467dc6961c9057380d9675a0ce10f1de00049a70101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200360ea000480d9675a087ffffe52000000000d2f6e6f64655374726174756d2fffffffff020003164e020000001976a914a4e47780f16cb0f4617946417edaa60a0077857388ac00e1f505000000001976a91452d162b995a7da28fc9c3512857f80d82e9a3b3488ac00000000" +} From 394be855af050157b6f4d51d6a9fa67be1bd238b Mon Sep 17 00:00:00 2001 From: cipig <32116761+cipig@users.noreply.github.com> Date: Tue, 30 Jan 2018 08:52:47 +0000 Subject: [PATCH 07/79] add BitcoinZ (BTCZ) (#360) * add BitcoinZ (BTCZ) --- lib/coins.py | 16 ++++++++++++++++ tests/blocks/bitcoinz_mainnet_1000.json | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/blocks/bitcoinz_mainnet_1000.json diff --git a/lib/coins.py b/lib/coins.py index 1e426b280..aaf8d9ad1 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -881,6 +881,22 @@ class Zcash(EquihashMixin, Coin): RPC_PORT = 8232 REORG_LIMIT = 800 +class BitcoinZ(EquihashMixin, Coin): + NAME = "BitcoinZ" + SHORTNAME = "BTCZ" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("1CB8") + P2SH_VERBYTES = [bytes.fromhex("1CBD")] + WIF_BYTE = bytes.fromhex("80") + GENESIS_HASH = ('f499ee3d498b4298ac6a64205b8addb7' + 'c43197e2a660229be65db8a4534d75c1') + DESERIALIZER = lib_tx.DeserializerZcash + TX_COUNT = 171976 + TX_COUNT_HEIGHT = 81323 + TX_PER_BLOCK = 3 + RPC_PORT = 1979 + REORG_LIMIT = 800 + class Hush(EquihashMixin, Coin): NAME = "Hush" SHORTNAME = "HUSH" diff --git a/tests/blocks/bitcoinz_mainnet_1000.json b/tests/blocks/bitcoinz_mainnet_1000.json new file mode 100644 index 000000000..d2081afb7 --- /dev/null +++ b/tests/blocks/bitcoinz_mainnet_1000.json @@ -0,0 +1,14 @@ +{ + "hash": "0000006595cd8958c67f926ef65538be2091f433f9aeea2ad9d47851a4b1f40e", + "size": 1622, + "height": 1000, + "merkleroot": "2600e65ba7213deba961e41547e5d147853813a74acfeb55c918bae7584912a7", + "tx": [ + "2600e65ba7213deba961e41547e5d147853813a74acfeb55c918bae7584912a7" + ], + "time": 1505123523, + "nonce": "020000000000000000000000000000000000000000000000085e2b8902000048", + "bits": "1e011428", + "previousblockhash": "000000917c6a3f21d1f140fb7fdf2d736502a85d5f5fbe65366da0cc7d002fcc", + "block": "04000000cc2f007dcca06d3665be5f5f5da80265732ddf7ffb40f1d1213f6a7c91000000a7124958e7ba18c955ebcf4aa713388547d1e54715e461a9eb3d21a75be600260000000000000000000000000000000000000000000000000000000000000000c35cb6592814011e48000002892b5e08000000000000000000000000000000000000000000000002fd400500532fef985b9ac6eff0a13c2bdcd68cc0438f2db21c419d7f26c791458cc441ebd3315ae493e06c0f9d11aacb22304cce9c8b72346f7d30c846b9719a97994b4035c5af2120330e24468d0ada86520eddfaec21019140f6599c2a0d2b7e830cc2e35aeccc2891f8e20b67eddb6547fa8df391a6a2d37f8546b5371dc7c81f3caa01d44975fd21d632879397cdf0c1860698ce3617fead0657892d510146952fdb0fce15ea32d93c073cd2cc11c3600bc24644dc33efc6b324f13b60df3f36bac64e9080db2995d4db4b3b62b56beb11c6d0261ccbb381cbb3e79212078c3074b9de66cf7d73c3363c3e2981d3061d88db53784efcff96a4abbaafb317631de298a8ba1db6ffb1d903b21fd1238b1c2cc51addd5aa3589df10abe7368fa4dbc6c1e0fc329c6623d2444ef56b61e1a45183acc62b503a511cbc21643db68f3925515305d9f3446b1df78b1375911ff8c1064a2799f1f56623e9fb6274cd372c0a5b671a97d73aa5bf6e2dafcba3a83147431976392674ae7b5b400c2e20eca6c9fd4db26d2644c35c0ac359755bf4b8136f6f5d290c1218639c226cddc0df04ffa2e88c5b1547375a3d12e873cb0fc2d526dbec82070f5dbe0919cf87be2ce8c43dbfae356a187f38dfd2f9feb2072102af614f5159508fb6f21cce3a107ea7bc58531b5649ce2baadcdb618ad327b10b5f8922f3dab81a6b0b3bc0a1ed107572bccc70be67484460a9250fe68f112461172192219ae9f513c880cc85822663d85a321688054706b49b73b05931916b9a0340b5ef3c84cc31b4f18edd8c7460dfce787803f8e9c69038fa379e0eb8d205e9db4265820cd3c52add3f61114b9dfad720a7fa96396199f38e18f233cb9c8034af7abf5ee31918dfce18713ffff012036214ef10b90fea6cd96943bc946f9312b31f2c221517f82ba505532b1f9fc10059d84e47c7badd6983c0b9b8dd0ab03ece731310058fb1fc216a92edcd7e923d4fcc2055c6175f2a6b02e0d484865e4513ad16912f07a8bf72035d95171f171c62d210909ddddc4715db2bb6ca5e9eb17a75e008ba1f20f72871f35e6b626a5a570aa9cfcaae93175ad5caebd5e19bcdb023473732e2c113a9609ef071167f6ac2485b726942d9e61610dd29fdc23d2faabf3b122db6576e2575bbb873b14a4c6c11117dbfd329038132b1321f29fdca75d0b91e75b9c556cb5f16ee1042d7fa9149eb6a948ce38cdd2fda6e9b18dd8c6e0b32a887ccd61a21324ba335dfff6909f2a751a210195bfb3cea599b67a5b152a6ab33706923b39ea86004119e5d5152e063e049305fb7dd831743a89d055a17c569d4d592c0ffbcc8d45af1b86c795fa1da5d8e08699b20a4871a48d8e6f9a6fef477c784717f943451befeb3bd5d222363a4254b7755cafb4cebbdd5510305c8f602db67cb3bc7c1b14fef6a89a01aee53b1153735b52e52f6fba098242877233905dd61dd179240d69f8628d2cea79f61a72bb6725d95d8157a77f462aaa522c41eb9c5fabcc63ebbec09b993671ef93903a8cd66bccc03e4dad5a2767871623af042d839fa15f4ce9237eb81a5f6baf3eea3c551e641f0b891e11e16258f0a5ab837b3bae89a71c6f73e9f7a9946507a344d9d3a695e53e215d84d36d10b326199b4ed630505cc1508a792eff5dfa54377b518c5e84491938437464d5bee4df0c55b6355efdc768c39ab56d27eea0b3a9a65821c4f52fccc8260ddb51634a973148cfd7cd50d5336a843e3f9118aa09a5d6ff354705cff8108c0ef3529d566cb2eedaa38e1ed38af136f7c2f6b14b351bc6f96792b682ab324b327f865dc4a5b47e410c2f2017d244593ba4bf4d70aaee6d9d9465fc7db42f09e9ebd5aec0bad1c94f590324ede4da0f41cc90101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff3102e803005a2d4e4f4d50212068747470733a2f2f6769746875622e636f6d2f6a6f7368756179616275742f7a2d6e6f6d70ffffffff010054ce09230100001976a9149ca39c1d4d57d00ce10d43ab3c63bd49a871ad8888ac00000000" +} From 302cfcfebf17e079681b14951285d1166ae6ddb2 Mon Sep 17 00:00:00 2001 From: cipig <32116761+cipig@users.noreply.github.com> Date: Wed, 31 Jan 2018 08:01:33 +0000 Subject: [PATCH 08/79] add Bitcore (BTX) (#362) --- lib/coins.py | 12 ++++++++++++ tests/blocks/bitcore_mainnet_100000.json | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/blocks/bitcore_mainnet_100000.json diff --git a/lib/coins.py b/lib/coins.py index aaf8d9ad1..b4c356645 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1314,3 +1314,15 @@ class Newyorkcoin(AuxPowMixin, Coin): TX_COUNT_HEIGHT = 3948743 TX_PER_BLOCK = 2 REORG_LIMIT = 2000 + +class Bitcore(BitcoinMixin, Coin): + NAME = "Bitcore" + SHORTNAME = "BTX" + DESERIALIZER = lib_tx.DeserializerSegWit + GENESIS_HASH = ('604148281e5c4b7f2487e5d03cd60d8e' + '6f69411d613f6448034508cea52e9574') + TX_COUNT = 126979 + TX_COUNT_HEIGHT = 126946 + TX_PER_BLOCK = 2 + RPC_PORT = 8556 + diff --git a/tests/blocks/bitcore_mainnet_100000.json b/tests/blocks/bitcore_mainnet_100000.json new file mode 100644 index 000000000..4a1ec5592 --- /dev/null +++ b/tests/blocks/bitcore_mainnet_100000.json @@ -0,0 +1,15 @@ +{ + "hash": "99103dc00414fd4c90b03cd1a770626e2ac681baa10e5f7f7ce089a6d98f0dad", + "size": 415, + "height": 100000, + "merkleroot": "59ea3a554c4cad8ae14ef7d8eed6c1e4d339bacb9ea91e308583c41645b8eafa", + "tx": [ + "57a9b42c52a0886ad9eddd9e52ccf25a1e11cce1a95b413fb58a307c09f1249b", + "58e78239d2cfcd9d6d2da2e1f9d88b658919a74d2f1c298cc6421fd51f6256af" + ], + "time": 1513253979, + "nonce": 3609159125, + "bits": "1b135157", + "previousblockhash": "19568daa07c7e0b6da2afac1b595f10ffe22abf8c8be1937a64c3e788064984f", + "block": "000000204f986480783e4ca63719bec8f8ab22fe0ff195b5c1fa2adab6e0c707aa8d5619faeab84516c48385301ea99ecbba39d3e4c1d6eed8f74ee18aad4c4c553aea595b6c325a5751131bd5651fd70202000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1803a08601045b6c325a0881000067010000007969696d7000000000000139baa012000000001976a914f89644903c1a800b2906a86455138cf99ace9b7a88ac0000000002000000014293bea25fd7ba75b0a1bc7a771f1f5698dcd53c04d320f855357429d71f80b8000000006a473044022052496da791f8df38a36e2e7b1e54c41e13d8b2375d36930dbbdb4d08bf07d3e402206e6e87344d76205fff8f6852298f18ff1d10ade9df34c96b09a4c27b795be833012102c21bcdeb21fac4e71332063156d54bcedfed93a6eb30c131b083347ff05da5fcfeffffff02808d8aae000000001976a914347fd1a28f968648d0c4a75b0bb53eb4223a68c888ac65d9368c000000001976a914d00e25fe39beb4e38a4cb1a14b72c77040f096c888ac9f860100" +} From 900b1f59ec7d73f8732a6e25466d20286a36af00 Mon Sep 17 00:00:00 2001 From: FujiCoin Date: Tue, 6 Feb 2018 13:52:48 +0900 Subject: [PATCH 09/79] modify Fujicoin (#367) --- lib/coins.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index b4c356645..420ce28ec 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1165,7 +1165,6 @@ class Fujicoin(Coin): 'a636f70856183086842667a1597714a0') ESTIMATE_FEE = 0.001 RELAY_FEE = 0.001 - DAEMON = daemon.FakeEstimateFeeDaemon TX_COUNT = 170478 TX_COUNT_HEIGHT = 1521676 TX_PER_BLOCK = 1 From 63733223fd61d3fdabbdf1a95e2894ff98bffa3c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Fri, 9 Feb 2018 18:43:08 +0800 Subject: [PATCH 10/79] Update docs for blockchain.headers.subscribe --- docs/PROTOCOL.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index a8bf59666..c84a076cd 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -434,14 +434,25 @@ Subscribe to receive block headers when a new block is found. **Response** - The *deserialized header* [2]_ of the current block. + The *deserialized header* [2]_ of the current block chain tip. **Notification Parameters** As this is a subcription, the client will receive a notification when a new block is found. The parameters are: - [**header**] + [**deserialized_header**] + + **NOTE**: if a new block comes in quickly so the server has not + finished processing the prior block(s), it may skip them and only + notify of the new tip. The protocol does not guarantee + notifications of all intermediate blocks. + + In a similar vein, the client also needs to be able to handle chain + reorganisations; in case of a re-org the new tip will not connect + directly onto the prior chain tip. The client needs to be able to + figure out where the connection point and request any missing block + headers. blockchain.numblocks.subscribe From 22c75a6216752c202e803f8ac8ed5397ce84deca Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 11 Feb 2018 22:10:07 +0800 Subject: [PATCH 11/79] Remove deprecated "with await" syntax --- server/block_processor.py | 4 ++-- server/db.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/block_processor.py b/server/block_processor.py index 2b2840d9c..ea4265e91 100644 --- a/server/block_processor.py +++ b/server/block_processor.py @@ -64,7 +64,7 @@ async def reset_height(self): Used in blockchain reorganisations. This coroutine can be called asynchronously to the _prefetch coroutine so we must synchronize with a semaphore.''' - with await self.semaphore: + async with self.semaphore: self.fetched_height = self.bp.height self.refill_event.set() @@ -85,7 +85,7 @@ async def _prefetch_blocks(self): ''' daemon = self.bp.daemon daemon_height = await daemon.height(self.bp.caught_up_event.is_set()) - with await self.semaphore: + async with self.semaphore: while self.cache_size < self.min_cache_size: # Try and catch up all blocks but limit to room in cache. # Constrain fetch count to between 0 and 500 regardless; diff --git a/server/db.py b/server/db.py index 46aef3f5f..9b1cc6bf5 100644 --- a/server/db.py +++ b/server/db.py @@ -662,7 +662,7 @@ async def compact_history(self, loop): while self.comp_cursor != -1: if self.semaphore.locked: self.log_info('compact_history: waiting on semaphore...') - with await self.semaphore: + async with self.semaphore: await loop.run_in_executor(None, self._compact_history, limit) def cancel_history_compaction(self): From fe90dfaf7cba48af875ece5868fed437adddcc96 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 11 Feb 2018 22:28:45 +0800 Subject: [PATCH 12/79] Update Bicoin Cash server list --- lib/coins.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index 420ce28ec..e08de8dfc 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -403,14 +403,14 @@ class BitcoinCash(BitcoinMixin, Coin): TX_COUNT_HEIGHT = 479636 TX_PER_BLOCK = 50 PEERS = [ - 'electroncash.bitcoinplug.com s t', + 'electroncash.bitcoinplug.com s t', # 1 strike 'electrum-abc.criptolayer.net s50012', 'electroncash.cascharia.com s50002', 'bch.arihanc.com t52001 s52002', - 'mash.1209k.com s t', - 'h.1209k.com s t', + 'jelectrum-cash.1209k.com s t', 'abc.vom-stausee.de t52001 s52002', 'abc1.hsmiths.com t60001 s60002', + 'electroncash.checksum0.com s t', ] @@ -1324,4 +1324,3 @@ class Bitcore(BitcoinMixin, Coin): TX_COUNT_HEIGHT = 126946 TX_PER_BLOCK = 2 RPC_PORT = 8556 - From 3e6ced6039e9f47ff2f54544130c09a5555b5f40 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 11 Feb 2018 22:39:01 +0800 Subject: [PATCH 13/79] Minor documentation updates --- README.rst | 4 +--- docs/PROTOCOL.rst | 41 ++++++++++++++--------------------------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 7b7b10fcd..4ae380351 100644 --- a/README.rst +++ b/README.rst @@ -33,9 +33,7 @@ Features time to height 448k (mid January 2017) reported is under 4h 30m. On the same hardware JElectrum would take around 4 days and electrum-server probably around 1 month. -- The full Electrum protocol is implemented. The only exception is - the blockchain.address.get_proof RPC call, which is not used by - Electrum GUI clients, and can only be invoked from the command line. +- The full current Electrum protocol is implemented. - Various configurable means of controlling resource consumption and handling denial of service attacks. These include maximum connection counts, subscription limits per-connection and across all diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index c84a076cd..eb928d739 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -5,11 +5,11 @@ Electrum Protocol This is intended to be a reference for client and server authors alike. -I have attempted to ensure what is written is correct for the three -known server implementations: electrum-server, jelectrum and -ElectrumX, and also for Electrum clients of the 2.x series. We know -other clients exist but I am not aware of the source of any being -publicly available. +I have attempted to ensure what is written is correct for the two +known remaining server implementations: jelectrum and ElectrumX, and +also for Electrum clients of the 2.x series. We know other clients +exist but I am not aware of the source of any being publicly +available. Message Stream @@ -25,14 +25,14 @@ requests should limit their size depending on the nature of their query, because servers will limit response size as an anti-DoS mechanism. -RPC calls and responses are separated by newlines in the stream. The -JSON specification does not permit control characters within strings, -so no confusion is possible there. However it does permit newlines as -extraneous whitespace between elements; client and server MUST NOT use -newlines in such a way. +Eeach RPC call, and each response, is separated by a single newline in +their respective streams. The JSON specification does not permit +control characters within strings, so no confusion is possible there. +However it does permit newlines as extraneous whitespace between +elements; client and server MUST NOT use newlines in such a way. If using JSON RPC 2.0's feature of parameter passing by name, the -names shown in the protocol versions's description MUST be used. +names shown in the protocol version's description MUST be used. A server advertising support for a particular protocol version MUST support each method documented for that protocol version, unless the @@ -41,6 +41,7 @@ additional parameters with unspecified behaviour. Use of additional parameters is discouraged as it may conflict with future versions of the protocol. + Notifications ------------- @@ -257,19 +258,6 @@ Return the unconfirmed transactions of a bitcoin address. ] -blockchain.address.get_proof -============================ - -This method is optional and deprecated, and hence its response will -not be described here. - - blockchain.address.get_proof(**address**) - - **address** - - The address as a Base58 string. - - blockchain.address.listunspent ============================== @@ -681,9 +669,8 @@ following changes: * improved semantics of `server.version` to aid protocol negotiation, and a changed return value. -* version 1.0 methods `blockchain.address.get_proof`, - `blockchain.utxo.get_address` and `blockchain.numblocks.subscribe` - have been removed. +* version 1.0 methods `blockchain.utxo.get_address` + and `blockchain.numblocks.subscribe` have been removed. * method `blockchain.transaction.get` no longer takes the *height* argument that was ignored in 1.0, providing one will return an error. From 76f4969a9840a5918e70b6ffd3a0b1e5b0fd9686 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sun, 11 Feb 2018 23:21:30 +0800 Subject: [PATCH 14/79] listunspent methods consider mempool receipts - Update docs. Height is 0 for mempool receipts - Implement mempool.get_utxos() and use it - Rename mempool.spends to mempool.potential_spends Closes #365 --- docs/PROTOCOL.rst | 17 ++++++++++------- server/controller.py | 11 ++++++----- server/mempool.py | 29 ++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index eb928d739..9e0cf49bb 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -272,14 +272,17 @@ Return an ordered list of UTXOs sent to a bitcoin address. **Response** A list of unspent outputs in blockchain order. Each transaction - is a dictionary with keys *height* , *tx_pos*, *tx_height* and + is a dictionary with keys *height* , *tx_pos*, *tx_hash* and *value* keys. *height* is the integer height of the block the - transaction was confirmed in; if unconfirmed then *height* is 0 if - all inputs are confirmed, and -1 otherwise. *tx_hash* the - transaction hash in hexadecimal, *tx_pos* the zero-based index of - the output in the transaction's list of outputs, and *value* its - integer value in minimum coin units (satoshis in the case of - Bitcoin). + transaction was confirmed in, *tx_hash* the transaction hash in + hexadecimal, *tx_pos* the zero-based index of the output in the + transaction's list of outputs, and *value* its integer value in + minimum coin units (satoshis in the case of Bitcoin). + + This function takes the mempool into account. Mempool + transactions paying to the address are included at the end of the + list in an undefined order, each with *tx_height* of zero. Any + output that is spent in the mempool does not appear. **Response Example** diff --git a/server/controller.py b/server/controller.py index 57b80c74b..2ecea1785 100644 --- a/server/controller.py +++ b/server/controller.py @@ -788,15 +788,16 @@ async def scripthash_get_mempool(self, scripthash): return await self.unconfirmed_history(hashX) async def hashX_listunspent(self, hashX): - '''Return the list of UTXOs of a script hash. - - We should remove mempool spends from the in-DB UTXOs.''' + '''Return the list of UTXOs of a script hash, including mempool + effects.''' utxos = await self.get_utxos(hashX) - spends = await self.mempool.spends(hashX) + utxos = sorted(utxos) + utxos.extend(self.mempool.get_utxos(hashX)) + spends = await self.mempool.potential_spends(hashX) return [{'tx_hash': hash_to_str(utxo.tx_hash), 'tx_pos': utxo.tx_pos, 'height': utxo.height, 'value': utxo.value} - for utxo in sorted(utxos) + for utxo in utxos if (utxo.tx_hash, utxo.tx_pos) not in spends] async def address_listunspent(self, address): diff --git a/server/mempool.py b/server/mempool.py index 286539ce9..2cef7f8da 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -305,14 +305,33 @@ async def transactions(self, hashX): item = self.txs.get(hex_hash) if not item or not raw_tx: continue - txin_pairs, txout_pairs, tx_fee, tx_size = item + tx_fee = item[2] tx = deserializer(raw_tx).read_tx() unconfirmed = any(hash_to_str(txin.prev_hash) in self.txs for txin in tx.inputs) result.append((hex_hash, tx_fee, unconfirmed)) return result - async def spends(self, hashX): + def get_utxos(self, hashX): + '''Return an unordered list of UTXO named tuples from mempool + transactions that pay to hashX. + + This does not consider if any other mempool transactions spend + the outputs. + ''' + utxos = [] + # hashXs is a defaultdict, so use get() to query + for hex_hash in self.hashXs.get(hashX, []): + item = self.txs.get(hex_hash) + if not item: + continue + txout_pairs = item[1] + for pos, (hX, value) in enumerate(txout_pairs): + if hX == hashX: + utxos.append(UTXO(-1, pos, hex_hash, 0, value)) + return utxos + + async def potential_spends(self, hashX): '''Return a set of (prev_hash, prev_idx) pairs from mempool transactions that touch hashX. @@ -320,14 +339,14 @@ async def spends(self, hashX): ''' deserializer = self.coin.DESERIALIZER pairs = await self.raw_transactions(hashX) - spends = set() + result = set() for hex_hash, raw_tx in pairs: if not raw_tx: continue tx = deserializer(raw_tx).read_tx() for txin in tx.inputs: - spends.add((txin.prev_hash, txin.prev_idx)) - return spends + result.add((txin.prev_hash, txin.prev_idx)) + return result def value(self, hashX): '''Return the unconfirmed amount in the mempool for hashX. From d075ebba862f313bc19b662024a7278fefaadc9d Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Mon, 12 Feb 2018 11:44:26 +0800 Subject: [PATCH 15/79] Add missing import Closes #370 --- server/mempool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/mempool.py b/server/mempool.py index 2cef7f8da..8510d504c 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -15,6 +15,7 @@ from lib.hash import hash_to_str, hex_str_to_hash import lib.util as util from server.daemon import DaemonError +from server.db import UTXO class MemPool(util.LoggedClass): From 790385cf7b4ed1ed877bfe6669097ad037860e5e Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 14 Feb 2018 22:39:53 +0800 Subject: [PATCH 16/79] UTXO holds a binary hash Fixes #371 --- server/mempool.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/mempool.py b/server/mempool.py index 8510d504c..4236f066a 100644 --- a/server/mempool.py +++ b/server/mempool.py @@ -329,7 +329,9 @@ def get_utxos(self, hashX): txout_pairs = item[1] for pos, (hX, value) in enumerate(txout_pairs): if hX == hashX: - utxos.append(UTXO(-1, pos, hex_hash, 0, value)) + # Unfortunately UTXO holds a binary hash + utxos.append(UTXO(-1, pos, hex_str_to_hash(hex_hash), + 0, value)) return utxos async def potential_spends(self, hashX): From dfea542d0eceda17d9a3ba1b546419c7e7eb402d Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Wed, 21 Feb 2018 11:22:16 +0100 Subject: [PATCH 17/79] Add Bitcoin Atom support (#378) --- lib/coins.py | 40 ++++++++++++++++++++ lib/tx.py | 11 ++++++ tests/blocks/bitcoinatom_mainnet_586900.json | 15 ++++++++ tests/blocks/bitcoinatom_mainnet_586956.json | 15 ++++++++ 4 files changed, 81 insertions(+) create mode 100644 tests/blocks/bitcoinatom_mainnet_586900.json create mode 100644 tests/blocks/bitcoinatom_mainnet_586956.json diff --git a/lib/coins.py b/lib/coins.py index e08de8dfc..eea54f10d 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1324,3 +1324,43 @@ class Bitcore(BitcoinMixin, Coin): TX_COUNT_HEIGHT = 126946 TX_PER_BLOCK = 2 RPC_PORT = 8556 + + +class BitcoinAtom(Coin): + NAME = "BitcoinAtom" + SHORTNAME = "BCA" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("17") + P2SH_VERBYTES = [bytes.fromhex("0a")] + WIF_BYTE = bytes.fromhex("80") + GENESIS_HASH = ('000000000019d6689c085ae165831e93' + '4ff763ae46a2a6c172b3f1b60a8ce26f') + STATIC_BLOCK_HEADERS = False + DESERIALIZER = lib_tx.DeserializerBitcoinAtom + HEADER_SIZE_POST_FORK = 84 + BLOCK_PROOF_OF_STAKE = 0x01 + BLOCK_PROOF_OF_STAKE_FLAGS = b'\x01\x00\x00\x00' + TX_COUNT = 295158744 + TX_COUNT_HEIGHT = 589197 + TX_PER_BLOCK = 10 + RPC_PORT = 9136 + REORG_LIMIT = 5000 + + @classmethod + def header_hash(cls, header): + '''Given a header return hash''' + header_to_be_hashed = header[:cls.BASIC_HEADER_SIZE] + # New block header format has some extra flags in the end + if len(header) == cls.HEADER_SIZE_POST_FORK: + flags, = struct.unpack('= self.FORK_BLOCK_HEIGHT: + header_len += 4 # flags + return self._read_nbytes(header_len) diff --git a/tests/blocks/bitcoinatom_mainnet_586900.json b/tests/blocks/bitcoinatom_mainnet_586900.json new file mode 100644 index 000000000..cd06171e3 --- /dev/null +++ b/tests/blocks/bitcoinatom_mainnet_586900.json @@ -0,0 +1,15 @@ +{ + "hash": "0000000000000000da3df1b019576a00d93974818b392b4c1a0a15f3df7c66fb", + "size": 510, + "height": 586900, + "merkleroot": "296a6318fb26dd39534acbe2f2c4cd82d0a9ce00d727d37deb0684748f7b73a7", + "tx": [ + "31417678bfbfb9784494974a1b5a7fc5b072c2fc1077f7951158cba43452d0af", + "253b8c15e7d5449dd604f07b5923a59ea35c6d0db0c0603497fec51cac317533" + ], + "time": 1518611740, + "nonce": 2909803309, + "bits": "1903fffc", + "previousblockhash": "0000000000000003060c045d260d64f50cbd03f827630974fc7686d7b0ed002f", + "block": "000000202f00edb0d78676fc74096327f803bd0cf5640d265d040c060300000000000000a7737b8f748406eb7dd327d700cea9d082cdc4f2e2cb4a5339dd26fb18636a291c2d845afcff03192d1370ad000000800201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200394f408041c2d845a0877ffb4bff7f26d020d2f6e6f64655374726174756d2f00000000020000000000000000266a24aa21a9ed370b337f1e15c7e0159715ea1707ec27b4a123f1a8810ad88e3b24ae412cd764288e814a000000001976a9143c0b217ab4208ab437dfc49c96d7d6f9e6608c2788ac000000000200000001af692cd7d8192a3b80f405e528d4cb4bc49efcad44c6d290a11cd21134a3bb82010000006a4730440220721a2935cc808b96ab20685c4bff77a1c253c5827c9fec4e7a54f2170c44870e02203fd847b15b449f3154acad12ed2f1d29b067cf73640e678a4b847d851441f4b0412103e0d0b2ea7fe2b07a3e26957a995190e4cbbd590653c8b118e161bcbcdbf52c4cfeffffff021d1db709000000001976a914035e3987787b71e772366787e087c8e9351dd1a888acb6d84c00000000001976a9143acea32e9ebc212a858c28a9624fb8924f35d8f288ac93f40800" +} \ No newline at end of file diff --git a/tests/blocks/bitcoinatom_mainnet_586956.json b/tests/blocks/bitcoinatom_mainnet_586956.json new file mode 100644 index 000000000..ab656413d --- /dev/null +++ b/tests/blocks/bitcoinatom_mainnet_586956.json @@ -0,0 +1,15 @@ +{ + "hash": "27b7efad91a595c0717a2e9fc4fb252545000c18a6610130d15382be83c402e3", + "size": 541, + "height": 586956, + "merkleroot": "81aa679f1878a93367dad062c8a6964847881260c19320b4e3664c63deabac7f", + "tx": [ + "789dda752eecf22056a93e8bcdf89eb31ac8102485124a7488fa0472250ac382", + "0f35c0b9bbf661787bfc920b5f4571440602e5723144c10b8fa5629dc3279abf" + ], + "time": 1518630315, + "nonce": 0, + "bits": "1d00ffff", + "previousblockhash": "0000000000000002c32c76bc7971dd10adf28abd4ff45c1b23c5b74d0265ffcc", + "block": "00000020ccff65024db7c5231b5cf44fbd8af2ad10dd7179bc762cc302000000000000007facabde634c66e3b42093c1601288474896a6c862d0da6733a978189f67aa81ab75845affff001d00000000010000800202000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0603ccf4080101ffffffff02ca000000000000002321034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ac0000000000000000266a24aa21a9ed164fe82b7473bf82cfdd7c28f90668f9fe78200022bafe159f293b4fe5e10832000000000200000001bef5797f980f07ba3bc619d2eef2ad728737709a747c4adf477dfb05186e7fe2010000006a47304402201fbef43b1877a013291effd9d9da882d128109d89db28ddd96f0bcb65ac4aab002206893640e754a9d147355474229af237581956fa191cb806648744e2d6ba989664121034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ffffffff01b67b814a000000002321034055f219c9619e18ef193c0def3e43ebffea827c950f0e78ee8f146b9f4c8102ac000000004630440220413932292b5e1cc5ae8cc019b11e5db7ec4432d260e8fee0f9c0a195a5aec28902204a90b318cb1b4f77ebd678db45fc0fce5ab3f0cc2d89aacaa523d8b6cd82f29e" +} \ No newline at end of file From b37b53f3585117e6d92fe8cef304bef8e55dd3d6 Mon Sep 17 00:00:00 2001 From: Lucas Betschart Date: Wed, 21 Feb 2018 11:53:07 +0100 Subject: [PATCH 18/79] Add Feathercoin test (#379) --- tests/blocks/feathercoin_mainnet_2001000.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/blocks/feathercoin_mainnet_2001000.json diff --git a/tests/blocks/feathercoin_mainnet_2001000.json b/tests/blocks/feathercoin_mainnet_2001000.json new file mode 100644 index 000000000..5d2587261 --- /dev/null +++ b/tests/blocks/feathercoin_mainnet_2001000.json @@ -0,0 +1,14 @@ +{ + "hash": "98b3e5827a174ee71c5922cfa4c7e290af86f5efa800cba972b5f147ab9dd2bc", + "size": 197, + "height": 2001000, + "merkleroot": "34f83ad3d320ffca8cac9bd41fdff03c9339c6277081071f35b5902dc7efcc84", + "tx": [ + "34f83ad3d320ffca8cac9bd41fdff03c9339c6277081071f35b5902dc7efcc84" + ], + "time": 1513018962, + "nonce": 2937419649, + "bits": "1c05b5e8", + "previousblockhash": "4cc309aa872da071f9c96b95382c2cc8d72e72b95407de6d7d23298b1da3bcca", + "block": "02000000cabca31d8b29237d6dde0754b9722ed7c82c2c38956bc9f971a02d87aa09c34c84ccefc72d90b5351f07817027c639933cf0df1fd49bac8ccaff20d3d33af83452d62e5ae8b5051c817715af0102000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1f0368881e062f503253482f0452d62e5a0881025d640c0000007969696d7000000000000100286bee000000001976a9141881bf27015e39946380a9d609a769e91548c8a488ac00000000" +} From 1c972d9592f2cd7e4994e8b7c31a6de34e8c4005 Mon Sep 17 00:00:00 2001 From: Johann Bauer Date: Thu, 22 Feb 2018 00:17:23 +0100 Subject: [PATCH 19/79] Only test on stable Python (#380) Since tests on nightly are quite likely to fail anyway, it doesn't make too much sense to test there. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6612766fe..e18dcc062 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ before_install: - sudo apt-get install -yq libleveldb-dev librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev python: - "3.6" - - "nightly" # command to install dependencies install: - pip install aiohttp From 7a989ad4e7dd949a9393a68dc70db3f7dbc91f67 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Thu, 22 Feb 2018 15:35:13 +0800 Subject: [PATCH 20/79] Remove get_chunk restriction --- server/session.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/session.py b/server/session.py index 91ba12ac9..7f2508fd2 100644 --- a/server/session.py +++ b/server/session.py @@ -112,7 +112,6 @@ def __init__(self, *args, **kwargs): self.max_subs = self.env.max_session_subs self.hashX_subs = {} self.mempool_statuses = {} - self.chunk_indices = [] self.protocol_version = None self.set_protocol_handlers((1, 0)) @@ -260,15 +259,6 @@ def block_get_chunk(self, index): index: the chunk index''' index = self.controller.non_negative_integer(index) - if self.client_version < (2, 8, 3): - self.chunk_indices.append(index) - self.chunk_indices = self.chunk_indices[-5:] - # -2 allows backing up a single chunk but no more. - if index <= max(self.chunk_indices[:-2], default=-1): - msg = ('chunk indices not advancing (wrong network?): {}' - .format(self.chunk_indices)) - # use INVALID_REQUEST to trigger a disconnect - raise RPCError(msg, JSONRPC.INVALID_REQUEST) return self.controller.get_chunk(index) def is_tor(self): From 2f26b7322eb80d29062d802f772c5c0fbc212f1f Mon Sep 17 00:00:00 2001 From: Johann Bauer Date: Thu, 22 Feb 2018 15:42:30 +0100 Subject: [PATCH 21/79] Reenable nightly but without coverage (#383) --- .travis.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e18dcc062..32fd10aef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,13 @@ language: python before_install: - sudo add-apt-repository -y ppa:giskou/librocksdb - sudo apt-get -qq update - - sudo apt-get install -yq libleveldb-dev librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev + - sudo apt-get install -yq librocksdb libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev + - wget https://launchpad.net/ubuntu/+archive/primary/+files/leveldb_1.20.orig.tar.gz + - tar -xzvf leveldb_1.20.orig.tar.gz + - pushd leveldb-1.20 && make && sudo mv out-shared/libleveldb.* /usr/local/lib && sudo cp -R include/leveldb /usr/local/include && sudo ldconfig && popd python: - "3.6" + - "nightly" # command to install dependencies install: - pip install aiohttp @@ -14,7 +18,11 @@ install: - pip install plyvel - pip install pyrocksdb - pip install pytest-cov - - pip install python-coveralls # command to run tests script: pytest --cov=server --cov=lib --cov=wallet -after_success: coveralls +# Dont report coverage from nightly +after_success: + - if [[ $(python3 -V 2>&1) == *"Python 3.6"* ]]; then + pip install python-coveralls; + coveralls; + fi From 6add2e5a9b77efeaf6d4e18ce98087dfa315d390 Mon Sep 17 00:00:00 2001 From: "John L. Jegutanis" Date: Tue, 27 Feb 2018 14:44:30 +0100 Subject: [PATCH 22/79] Add custom base58check algorithm (#389) --- lib/coins.py | 10 ++++++---- lib/hash.py | 8 ++++---- tests/lib/test_addresses.py | 2 +- tests/lib/test_hash.py | 17 +++++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index eea54f10d..e1f96b34a 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -70,6 +70,8 @@ class Coin(object): BLOCK_PROCESSOR = BlockProcessor XPUB_VERBYTES = bytes('????', 'utf-8') XPRV_VERBYTES = bytes('????', 'utf-8') + ENCODE_CHECK = Base58.encode_check + DECODE_CHECK = Base58.decode_check # Peer discovery PEER_DEFAULT_PORTS = {'t': '50001', 's': '50002'} PEERS = [] @@ -168,7 +170,7 @@ def address_to_hashX(cls, address): def P2PKH_address_from_hash160(cls, hash160): '''Return a P2PKH address given a public key.''' assert len(hash160) == 20 - return Base58.encode_check(cls.P2PKH_VERBYTE + hash160) + return cls.ENCODE_CHECK(cls.P2PKH_VERBYTE + hash160) @classmethod def P2PKH_address_from_pubkey(cls, pubkey): @@ -179,7 +181,7 @@ def P2PKH_address_from_pubkey(cls, pubkey): def P2SH_address_from_hash160(cls, hash160): '''Return a coin address given a hash160.''' assert len(hash160) == 20 - return Base58.encode_check(cls.P2SH_VERBYTES[0] + hash160) + return cls.ENCODE_CHECK(cls.P2SH_VERBYTES[0] + hash160) @classmethod def multisig_address(cls, m, pubkeys): @@ -212,7 +214,7 @@ def pay_to_address_script(cls, address): Pass the address (either P2PKH or P2SH) in base58 form. ''' - raw = Base58.decode_check(address) + raw = cls.DECODE_CHECK(address) # Require version byte(s) plus hash160. verbyte = -1 @@ -233,7 +235,7 @@ def privkey_WIF(cls, privkey_bytes, compressed): payload = bytearray(cls.WIF_BYTE) + privkey_bytes if compressed: payload.append(0x01) - return Base58.encode_check(payload) + return cls.ENCODE_CHECK(payload) @classmethod def header_hash(cls, header): diff --git a/lib/hash.py b/lib/hash.py index 0f9436d91..47039bd3f 100644 --- a/lib/hash.py +++ b/lib/hash.py @@ -143,18 +143,18 @@ def encode(be_bytes): return txt[::-1] @staticmethod - def decode_check(txt): + def decode_check(txt, *, hash_fn=double_sha256): '''Decodes a Base58Check-encoded string to a payload. The version prefixes it.''' be_bytes = Base58.decode(txt) result, check = be_bytes[:-4], be_bytes[-4:] - if check != double_sha256(result)[:4]: + if check != hash_fn(result)[:4]: raise Base58Error('invalid base 58 checksum for {}'.format(txt)) return result @staticmethod - def encode_check(payload): + def encode_check(payload, *, hash_fn=double_sha256): """Encodes a payload bytearray (which includes the version byte(s)) into a Base58Check string.""" - be_bytes = payload + double_sha256(payload)[:4] + be_bytes = payload + hash_fn(payload)[:4] return Base58.encode(be_bytes) diff --git a/tests/lib/test_addresses.py b/tests/lib/test_addresses.py index e832f0476..293f2eb5d 100644 --- a/tests/lib/test_addresses.py +++ b/tests/lib/test_addresses.py @@ -60,7 +60,7 @@ def test_address_to_hashX(address): def test_address_from_hash160(address): coin, addr, hash, _ = address - raw = Base58.decode_check(addr) + raw = coin.DECODE_CHECK(addr) verlen = len(raw) - 20 assert verlen > 0 verbyte, hash_bytes = raw[:verlen], raw[verlen:] diff --git a/tests/lib/test_hash.py b/tests/lib/test_hash.py index f1b9d463f..87aa3558c 100644 --- a/tests/lib/test_hash.py +++ b/tests/lib/test_hash.py @@ -1,6 +1,7 @@ # # Tests of lib/hash.py # +from functools import partial import pytest @@ -66,3 +67,19 @@ def test_Base58_encode_check(): with pytest.raises(TypeError): lib_hash.Base58.encode_check('foo') assert lib_hash.Base58.encode_check(b'foo') == '4t9WKfuAB8' + +def test_Base58_decode_check_custom(): + decode_check_sha256 = partial(lib_hash.Base58.decode_check, + hash_fn=lib_hash.sha256) + with pytest.raises(TypeError): + decode_check_sha256(b'foo') + assert decode_check_sha256('4t9WFhKfWr') == b'foo' + with pytest.raises(lib_hash.Base58Error): + decode_check_sha256('4t9WFhKfWp') + +def test_Base58_encode_check_custom(): + encode_check_sha256 = partial(lib_hash.Base58.encode_check, + hash_fn=lib_hash.sha256) + with pytest.raises(TypeError): + encode_check_sha256('foo') + assert encode_check_sha256(b'foo') == '4t9WFhKfWr' From 4f871cec0ff3c6050002bf4d76118be78346e3a7 Mon Sep 17 00:00:00 2001 From: Lucas Betschart Date: Thu, 1 Mar 2018 18:20:10 +0100 Subject: [PATCH 23/79] Feathercoin: Fix P2SH (#391) --- lib/coins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coins.py b/lib/coins.py index e1f96b34a..30ed50b3d 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1288,7 +1288,7 @@ class Feathercoin(Coin): XPUB_VERBYTES = bytes.fromhex("0488BC26") XPRV_VERBYTES = bytes.fromhex("0488DAEE") P2PKH_VERBYTE = bytes.fromhex("0E") - P2SH_VERBYTES = [bytes.fromhex("32"), bytes.fromhex("05")] + P2SH_VERBYTES = [bytes.fromhex("05")] WIF_BYTE = bytes.fromhex("8E") GENESIS_HASH = ('12a765e31ffd4059bada1e25190f6e98' 'c99d9714d334efa41a195a7e7e04bfe2') From 49ee00834616f5bded1df5a57ac892993c9ff4fd Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 3 Mar 2018 17:23:50 +0800 Subject: [PATCH 24/79] Relax the constraints on read_headers They were really for fs_block_hashes; that still enforces the full constraint. Simplifies get_chunk. --- server/controller.py | 14 +++++++------- server/db.py | 30 ++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/server/controller.py b/server/controller.py index 2ecea1785..7379186b7 100644 --- a/server/controller.py +++ b/server/controller.py @@ -299,11 +299,11 @@ def notify_peers(self, updates): def electrum_header(self, height): '''Return the binary header at the given height.''' - if not 0 <= height <= self.bp.db_height: - raise RPCError('height {:,d} out of range'.format(height)) if height in self.header_cache: return self.header_cache[height] - header = self.bp.read_headers(height, 1) + header, n = self.bp.read_headers(height, 1) + if n != 1: + raise RPCError('height {:,d} out of range'.format(height)) header = self.coin.electrum_header(header, height) self.header_cache[height] = header return header @@ -750,10 +750,10 @@ def job(): def get_chunk(self, index): '''Return header chunk as hex. Index is a non-negative integer.''' chunk_size = self.coin.CHUNK_SIZE - next_height = self.bp.db_height + 1 - start_height = min(index * chunk_size, next_height) - count = min(next_height - start_height, chunk_size) - return self.bp.read_headers(start_height, count).hex() + start_height = index * chunk_size + count = chunk_size + headers, n = self.bp.read_headers(start_height, count).hex() + return headers # Client RPC "blockchain" command handlers diff --git a/server/db.py b/server/db.py index 9b1cc6bf5..22bfcf455 100644 --- a/server/db.py +++ b/server/db.py @@ -209,18 +209,25 @@ def fs_update(self, fs_height, headers, block_tx_hashes): offset = prior_tx_count * 32 self.hashes_file.write(offset, hashes) - def read_headers(self, start, count): - '''Requires count >= 0.''' + def read_headers(self, start_height, count): + '''Requires start_height >= 0, count >= 0. Reads as many headers as + are available starting at start_height up to count. This + would be zero if start_height is beyond self.db_height, for + example. + + Returns a (binary, n) pair where binary is the concatenated + binary headers, and n is the count of headers returned. + ''' # Read some from disk - disk_count = min(count, self.db_height + 1 - start) - if start < 0 or count < 0 or disk_count != count: + if start_height < 0 or count < 0: raise self.DBError('{:,d} headers starting at {:,d} not on disk' - .format(count, start)) + .format(count, start_height)) + disk_count = max(0, min(count, self.db_height + 1 - start_height)) if disk_count: - offset = self.header_offset(start) - size = self.header_offset(start + disk_count) - offset - return self.headers_file.read(offset, size) - return b'' + offset = self.header_offset(start_height) + size = self.header_offset(start_height + disk_count) - offset + return self.headers_file.read(offset, size), disk_count + return b'', 0 def fs_tx_hash(self, tx_num): '''Return a par (tx_hash, tx_height) for the given tx number. @@ -234,7 +241,10 @@ def fs_tx_hash(self, tx_num): return tx_hash, tx_height def fs_block_hashes(self, height, count): - headers_concat = self.read_headers(height, count) + headers_concat, headers_count = self.read_headers(height, count) + if headers_count != count: + raise self.DBError('only got {:,d} headers starting at {:,d}, not ' + '{:,d}'.format(headers_count, start, count)) offset = 0 headers = [] for n in range(count): From 7823129526818381beac40a0df9aec9227ff6ac7 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 3 Mar 2018 17:45:55 +0800 Subject: [PATCH 25/79] Clarify truncation behaviour of get_chunk in docs --- docs/PROTOCOL.rst | 5 ++++- server/session.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 9e0cf49bb..ea409a226 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -388,7 +388,10 @@ bandwidth-intensive request. **Response** The binary block headers, as hexadecimal strings, in order - concatenated together. + concatenated together. As many as headers as are available at + starting height 2016 * index will be returned; this may range from + 0 to 2016. + blockchain.estimatefee diff --git a/server/session.py b/server/session.py index 7f2508fd2..a487ed4c6 100644 --- a/server/session.py +++ b/server/session.py @@ -254,6 +254,26 @@ def server_features(self): '''Returns a dictionary of server features.''' return self.env.server_features() + def block_headers(self, start_height, count): + '''Return concatenated block headers as hex for the main chain; + count headers starting at start_height. + + start_height and count must be non-negative integers.''' + start_height = self.controller.non_negative_integer(start_height) + count = self.controller.non_negative_integer(count) + return self.controller.block_headers(start_height, count).hex() + + def block_get_chunk(self, index): + '''Return a chunk of block headers as a hexadecimal string. + + index: the chunk index''' + index = self.controller.non_negative_integer(index) + chunk_size = self.coin.CHUNK_SIZE + start_height = index * chunk_size + count = chunk_size + + return self.controller.block_headers(start_height, count).hex() + def block_get_chunk(self, index): '''Return a chunk of block headers as a hexadecimal string. From 5109990ea4888dc3a7085521fa23e8e24b6beced Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 3 Mar 2018 17:56:28 +0800 Subject: [PATCH 26/79] Add blockchain.block.headers RPC call --- docs/PROTOCOL.rst | 48 ++++++++++++++++++++++++++++++++++++++------ server/controller.py | 17 +++++++++------- server/session.py | 23 +++++++++++++-------- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index ea409a226..5f979d7c6 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -927,14 +927,50 @@ Get a list of features and services supported by the server. Protocol Version 1.2 -------------------- -Protocol version 1.2 is the same as version `1.1` except for the -addition of a new method `mempool.get_fee_histogram`. +Protocol version 1.2 introduces new methods `blockcahin.block.get_chunk`, +`mempool.get_fee_histogram`. -All methods with taking addresses are deprecated, and will be removed -at some point in the future. You should update your code to use -`Script Hashes`_ and the scripthash methods introduced in protocol 1.1 -instead. +`blockchain.block.get_chunk` and all methods beginning + `blockchain.address.` are deprecated and support will be removed in + some future protocol version. You should update your code to use + `blockchain.block.headers` and `Script Hashes`_ with the scripthash + methods introduced in protocol 1.1 instead. +blockchain.block.headers +======================== + +Return concatenated block headers as hexadecimal from the main chain. + + blockchain.block.headers(**start_height**, **count**) + + **start_height** + + The height of the first header requested, a non-negative integer. + + **count** + + The number of headers requested, a non-negative integer. + +**Response** + + A dictionary with at least 3 members: + +* **count** + + The number of headers returned, between zero and the number + requested. If the chain has not extended sufficiently far, only + the available headers will be returned. If more headers than + **max** were requested at most **max** will be returned. + +* **hex** + + The binary block headers concatenated together as hexadecimal + strings, in order. + +* **max** + + The maximum number of headers the server will return in a single + request. mempool.get_fee_histogram ========================= diff --git a/server/controller.py b/server/controller.py index 7379186b7..454de2b3e 100644 --- a/server/controller.py +++ b/server/controller.py @@ -747,13 +747,16 @@ def job(): return await self.run_in_executor(job) - def get_chunk(self, index): - '''Return header chunk as hex. Index is a non-negative integer.''' - chunk_size = self.coin.CHUNK_SIZE - start_height = index * chunk_size - count = chunk_size - headers, n = self.bp.read_headers(start_height, count).hex() - return headers + def block_headers(self, start_height, count): + '''Read count block headers starting at start_height; both + must be non-negative. + + The return value is (hex, n), where hex is the hex encoding of + the concatenated headers, and n is the number of headers read + (0 <= n <= count). + ''' + headers, n = self.bp.read_headers(start_height, count) + return headers.hex(), n # Client RPC "blockchain" command handlers diff --git a/server/session.py b/server/session.py index a487ed4c6..788e9e949 100644 --- a/server/session.py +++ b/server/session.py @@ -25,6 +25,8 @@ class SessionBase(JSONSession): sessions. ''' + MAX_CHUNK_SIZE = 2016 + def __init__(self, controller, kind): # Force v2 as a temporary hack for old Coinomi wallets # Remove in April 2017 @@ -255,13 +257,17 @@ def server_features(self): return self.env.server_features() def block_headers(self, start_height, count): - '''Return concatenated block headers as hex for the main chain; - count headers starting at start_height. + '''Return count concatenated block headers as hex for the main chain; + starting at start_height. - start_height and count must be non-negative integers.''' + start_height and count must be non-negative integers. At most + MAX_CHUNK_SIZE headers will be returned. + ''' start_height = self.controller.non_negative_integer(start_height) count = self.controller.non_negative_integer(count) - return self.controller.block_headers(start_height, count).hex() + count = min(count, self.MAX_CHUNK_SIZE) + hex_str, n = self.controller.block_headers(start_height, count) + return {'hex': hex_str, 'count': n, 'max': self.MAX_CHUNK_SIZE} def block_get_chunk(self, index): '''Return a chunk of block headers as a hexadecimal string. @@ -270,9 +276,8 @@ def block_get_chunk(self, index): index = self.controller.non_negative_integer(index) chunk_size = self.coin.CHUNK_SIZE start_height = index * chunk_size - count = chunk_size - - return self.controller.block_headers(start_height, count).hex() + hex_str, n = self.controller.block_headers(start_height, chunk_size) + return hex_str def block_get_chunk(self, index): '''Return a chunk of block headers as a hexadecimal string. @@ -462,7 +467,9 @@ def set_protocol_handlers(self, ptuple): if ptuple >= (1, 2): # New handler as of 1.2 handlers.update({ - 'mempool.get_fee_histogram': controller.mempool_get_fee_histogram, + 'mempool.get_fee_histogram': + controller.mempool_get_fee_histogram, + 'blockchain.block.headers': self.block_headers, }) self.electrumx_handlers = handlers From 5f5c5993398d3766d681984ac81e21c2301a5dc5 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Sat, 3 Mar 2018 18:45:04 +0800 Subject: [PATCH 27/79] Add example response to docs --- docs/PROTOCOL.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 5f979d7c6..d31235845 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -972,6 +972,17 @@ Return concatenated block headers as hexadecimal from the main chain. The maximum number of headers the server will return in a single request. +**Example Response** + +:: + + { + "count": 2, + "hex": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299'" + "max": 2016 + } + + mempool.get_fee_histogram ========================= From adf81138e38715faeb3affeba9e4b70599c57ff9 Mon Sep 17 00:00:00 2001 From: ghost43 Date: Sun, 4 Mar 2018 02:24:13 +0100 Subject: [PATCH 28/79] typos in PROTOCOL.rst (#394) --- docs/PROTOCOL.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index d31235845..173eba1e4 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -400,7 +400,7 @@ blockchain.estimatefee Return the estimated transaction fee per kilobyte for a transaction to be confirmed within a certain number of blocks. - blockchain.block.get_chunk(**number**) + blockchain.estimatefee(**number**) **number** @@ -555,7 +555,7 @@ blockchain.transaction.get_merkle Return the markle branch to a confirmed transaction given its hash and height. - blockchain.transaction.get(**tx_hash**, **height**) + blockchain.transaction.get_merkle(**tx_hash**, **height**) **tx_hash** @@ -927,7 +927,7 @@ Get a list of features and services supported by the server. Protocol Version 1.2 -------------------- -Protocol version 1.2 introduces new methods `blockcahin.block.get_chunk`, +Protocol version 1.2 introduces new methods `blockchain.block.headers`, `mempool.get_fee_histogram`. `blockchain.block.get_chunk` and all methods beginning From 19e0f9bce27ee24603e46c71265c60f8046537a7 Mon Sep 17 00:00:00 2001 From: Carsen Klock Date: Mon, 5 Mar 2018 03:26:25 -0700 Subject: [PATCH 29/79] Add Denarius (DNR) (#396) * Added DNR With tests --- .travis.yml | 1 + lib/coins.py | 39 ++++++++++++++++++++++++ setup.py | 1 + tests/blocks/denarius_mainnet_10000.json | 14 +++++++++ 4 files changed, 55 insertions(+) create mode 100644 tests/blocks/denarius_mainnet_10000.json diff --git a/.travis.yml b/.travis.yml index 32fd10aef..a76ba3234 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: - pip install ecdsa - pip install plyvel - pip install pyrocksdb + - pip install tribus-hash - pip install pytest-cov # command to run tests script: pytest --cov=server --cov=lib --cov=wallet diff --git a/lib/coins.py b/lib/coins.py index 30ed50b3d..dee9f089b 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -1232,6 +1232,45 @@ class CanadaeCoin(AuxPowMixin, Coin): TX_PER_BLOCK = 1 RPC_PORT = 34330 REORG_LIMIT = 1000 + +class Denarius(Coin): + NAME = "Denarius" + SHORTNAME = "DNR" + NET = "mainnet" + XPUB_VERBYTES = bytes.fromhex("0488b21e") + XPRV_VERBYTES = bytes.fromhex("0488ade4") + P2PKH_VERBYTE = bytes.fromhex("1E") #Address starts with a D + P2SH_VERBYTES = [bytes.fromhex("5A")] + WIF_BYTE = bytes.fromhex("9E") #WIF starts with a 6 + GENESIS_HASH = ('00000d5dbbda01621cfc16bbc1f9bf32' + '64d641a5dbf0de89fd0182c2c4828fcd') + DESERIALIZER = lib_tx.DeserializerTxTime + TX_COUNT = 4230 + RPC_PORT = 32339 + ESTIMATE_FEE = 0.00001 + RELAY_FEE = 0.00001 + DAEMON = daemon.FakeEstimateFeeDaemon + TX_COUNT_HEIGHT = 306187 + TX_PER_BLOCK = 4000 + + @classmethod + def header_hash(cls, header): + '''Given a header return the hash.''' + import tribus_hash + return tribus_hash.getPoWHash(header) + + +class DenariusTestnet(Denarius): + NET = "testnet" + XPUB_VERBYTES = bytes.fromhex("043587cf") + XPRV_VERBYTES = bytes.fromhex("04358394") + P2PKH_VERBYTE = bytes.fromhex("12") + P2SH_VERBYTES = [bytes.fromhex("74")] + WIF_BYTE = bytes.fromhex("ef") + GENESIS_HASH = ('000086bfe8264d241f7f8e5393f74778' + '4b8ca2aa98bdd066278d590462a4fdb4') + RPC_PORT = 32338 + REORG_LIMIT = 2000 class Sibcoin(Dash): diff --git a/setup.py b/setup.py index 6f7becf68..a3bfc2ac5 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ python_requires='>=3.6', # via environment variables, in which case I've tested with 15.0.4 # "x11_hash" package (1.4) is required to sync DASH network. + # "tribus_hash" package is required to sync Denarius network. install_requires=['plyvel', 'pylru', 'aiohttp >= 1'], packages=setuptools.find_packages(exclude=['tests']), description='ElectrumX Server', diff --git a/tests/blocks/denarius_mainnet_10000.json b/tests/blocks/denarius_mainnet_10000.json new file mode 100644 index 000000000..bbce5a357 --- /dev/null +++ b/tests/blocks/denarius_mainnet_10000.json @@ -0,0 +1,14 @@ +{ + "hash": "00000000058989f94843c6047bdf72b3f37f62055354c5ff859e28bb0b5282b9", + "size": 204, + "height": 10000, + "merkleroot": "65841bbf5c89ed2def56d4ef505ee1f680fd66028613e372ae7355f09b646a58", + "tx": [ + "65841bbf5c89ed2def56d4ef505ee1f680fd66028613e372ae7355f09b646a58" + ], + "time" : 1497949464, + "nonce" : 1484027400, + "bits" : "1c073cc2", + "previousblockhash": "0000000003db0426bcd14ad568df783dceaa1ad5def62d347fe6ef8d77c40a44", + "block": "06000000440ac4778defe67f342df6ded51aaace3d78df68d54ad1bc2604db0300000000586a649bf05573ae72e313860266fd80f6e15e50efd456ef2ded895cbf1b846518e54859c23c071c08767458010200000018e54859010000000000000000000000000000000000000000000000000000000000000000ffffffff170210270418e54859088100000a000000007969696d7000000000000100a3e11100000000232102ae784442221c0d8cc1db36d90ed2051bb08cb984575e8de3e57ddc209ef8b0a5ac0000000000" +} \ No newline at end of file From 5f3890879d0034a5fd473880477be8e0dabbe7b1 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 6 Mar 2018 20:57:37 +0800 Subject: [PATCH 30/79] Bitcoin Cash: remove dead server --- lib/coins.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/coins.py b/lib/coins.py index dee9f089b..2cfaf8d22 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -405,7 +405,6 @@ class BitcoinCash(BitcoinMixin, Coin): TX_COUNT_HEIGHT = 479636 TX_PER_BLOCK = 50 PEERS = [ - 'electroncash.bitcoinplug.com s t', # 1 strike 'electrum-abc.criptolayer.net s50012', 'electroncash.cascharia.com s50002', 'bch.arihanc.com t52001 s52002', @@ -1232,7 +1231,7 @@ class CanadaeCoin(AuxPowMixin, Coin): TX_PER_BLOCK = 1 RPC_PORT = 34330 REORG_LIMIT = 1000 - + class Denarius(Coin): NAME = "Denarius" SHORTNAME = "DNR" @@ -1252,7 +1251,7 @@ class Denarius(Coin): DAEMON = daemon.FakeEstimateFeeDaemon TX_COUNT_HEIGHT = 306187 TX_PER_BLOCK = 4000 - + @classmethod def header_hash(cls, header): '''Given a header return the hash.''' From 62220f64db45b29d930bdb0e4c9377593546d4cc Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 6 Mar 2018 22:25:54 +0800 Subject: [PATCH 31/79] Remove dead code left over from prior --- server/session.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/session.py b/server/session.py index 788e9e949..890fc8427 100644 --- a/server/session.py +++ b/server/session.py @@ -279,13 +279,6 @@ def block_get_chunk(self, index): hex_str, n = self.controller.block_headers(start_height, chunk_size) return hex_str - def block_get_chunk(self, index): - '''Return a chunk of block headers as a hexadecimal string. - - index: the chunk index''' - index = self.controller.non_negative_integer(index) - return self.controller.get_chunk(index) - def is_tor(self): '''Try to detect if the connection is to a tor hidden service we are running.''' From a3da48e7369186a30dfd776b765d33d589301c16 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 6 Mar 2018 23:37:57 +0900 Subject: [PATCH 32/79] Fix daemontools run script for dash shell Fixes #374 --- contrib/daemontools/run | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/daemontools/run b/contrib/daemontools/run index 773171b46..c279ecd4e 100755 --- a/contrib/daemontools/run +++ b/contrib/daemontools/run @@ -1,3 +1,5 @@ #!/bin/sh echo "Launching ElectrumX server..." -exec 2>&1 envdir ./env /bin/sh -c 'setuidgid $USERNAME python3 $ELECTRUMX' +USERNAME=$(envdir ./env printenv USERNAME) +ELECTRUMX=$(envdir ./env printenv ELECTRUMX) +ulimit -n 4000 && exec 2>&1 envdir ./env envuidgid $USERNAME python3 $ELECTRUMX From 521227c2b853b05ec6f4f388e06abf8f3bafde4c Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Tue, 6 Mar 2018 23:02:39 +0800 Subject: [PATCH 33/79] coin is on the controller, unfortunately --- server/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/session.py b/server/session.py index 890fc8427..e24f728c1 100644 --- a/server/session.py +++ b/server/session.py @@ -274,7 +274,7 @@ def block_get_chunk(self, index): index: the chunk index''' index = self.controller.non_negative_integer(index) - chunk_size = self.coin.CHUNK_SIZE + chunk_size = self.controller.coin.CHUNK_SIZE start_height = index * chunk_size hex_str, n = self.controller.block_headers(start_height, chunk_size) return hex_str From d5d5df54d873c2dbf70eddbf7e95780e9b9ef533 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Tue, 6 Mar 2018 17:30:46 +0200 Subject: [PATCH 34/79] electrum_server: set logging format (#398) * electrum_server: set logging format * daemontools: remove 't' multilog argument --- contrib/daemontools/log/run | 2 +- electrumx_server.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/daemontools/log/run b/contrib/daemontools/log/run index c12b043aa..b501bde2a 100755 --- a/contrib/daemontools/log/run +++ b/contrib/daemontools/log/run @@ -1,2 +1,2 @@ #!/bin/sh -exec multilog t s500000 n10 /path/to/log/dir +exec multilog s500000 n10 /path/to/log/dir diff --git a/electrumx_server.py b/electrumx_server.py index 77c7c6e58..4c4fa533e 100755 --- a/electrumx_server.py +++ b/electrumx_server.py @@ -18,7 +18,9 @@ def main(): '''Set up logging and run the server.''' - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)-9s %(message)-100s ' + '%(name)s [%(filename)s:%(lineno)d]') logging.info('ElectrumX server starting') try: controller = Controller(Env()) From 824dc76d95ec382480b5b7494468cf9058dac930 Mon Sep 17 00:00:00 2001 From: blackjok3rtt <30971146+blackjok3rtt@users.noreply.github.com> Date: Tue, 6 Mar 2018 23:34:17 +0800 Subject: [PATCH 35/79] Add coin SNG (#400) * add SnowGem --- lib/coins.py | 16 ++++++++++++++++ tests/blocks/snowgem_mainnet_102802.json | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/blocks/snowgem_mainnet_102802.json diff --git a/lib/coins.py b/lib/coins.py index 2cfaf8d22..80f4094cb 100644 --- a/lib/coins.py +++ b/lib/coins.py @@ -882,6 +882,22 @@ class Zcash(EquihashMixin, Coin): RPC_PORT = 8232 REORG_LIMIT = 800 +class SnowGem(EquihashMixin, Coin): + NAME = "SnowGem" + SHORTNAME = "SNG" + NET = "mainnet" + P2PKH_VERBYTE = bytes.fromhex("1C28") + P2SH_VERBYTES = [bytes.fromhex("1C2D")] + WIF_BYTE = bytes.fromhex("80") + GENESIS_HASH = ('00068b35729d9d2b0c294ff1fe9af009' + '4740524311a131de40e7f705e4c29a5b') + DESERIALIZER = lib_tx.DeserializerZcash + TX_COUNT = 140698 + TX_COUNT_HEIGHT = 102802 + TX_PER_BLOCK = 2 + RPC_PORT = 16112 + REORG_LIMIT = 800 + class BitcoinZ(EquihashMixin, Coin): NAME = "BitcoinZ" SHORTNAME = "BTCZ" diff --git a/tests/blocks/snowgem_mainnet_102802.json b/tests/blocks/snowgem_mainnet_102802.json new file mode 100644 index 000000000..11e86cad5 --- /dev/null +++ b/tests/blocks/snowgem_mainnet_102802.json @@ -0,0 +1,14 @@ +{ + "hash": "00000003f696e24d5fa933d1a839f3a972209e1c4abc5d236fa715fb1fee4532", + "size": 1655, + "height": 102802, + "merkleroot": "a93b43eecef263eb7e8892d4433cd24bafaa4e48c220714a7d99f867cb350a83", + "tx": [ + "a93b43eecef263eb7e8892d4433cd24bafaa4e48c220714a7d99f867cb350a83" + ], + "time": 1520332521, + "nonce": "0300000000000000000000000000000000000000000000000075e7d500000030", + "bits": "1d2671d8", + "previousblockhash": "000000067fd18074348b3a4e756552825dc190323185a1d7a5b37d071584c06e", + "block": "040000006ec08415077db3a5d7a185313290c15d825265754e3a8b347480d17f06000000830a35cb67f8997d4a7120c2484eaaaf4bd23c43d492887eeb63f2ceee433ba90000000000000000000000000000000000000000000000000000000000000000e96e9e5ad871261d30000000d5e77500000000000000000000000000000000000000000000000003fd4005005ecbcbb24c640cfae740fabcb1df353b01cad2f9320f4469b9135e96ae468435f84a32d61a021bcab8020641ed2e083a7777295352fa66c98b9cd97e8f2e27c8bd9e93cce9ea69aa25122f589e618623df834d19274c07d6879d39135943ac33767a8b6c029c4b6f2ec8a49b592057433c4ce4a9c3c2a182f97d7b00e42c1cd560e2a31e733224a5625adee23ebe3cd9bb4f485bad635517df991412d84fb0f79036812dfaac8908b7237481470d37d62e53e810ae1e36b3281ecc7c0d952d1dfd43cfc2db0f185c96c410122c37f9f565215aebdf2f192e48e45e43c779f0bf0e19261abbac3190c504cde573438615d58f6bf9eb878d6bddd75d0db1db0204c3f6cc90c3534da14d9e423172d9852846f334a687a5243b62c435964eff376efe97baad0833d08e136858cc23d0c85be7cee8859bb2abddc7cd3b3c9ab413172f8167ec362fdebecaa2717abdc331065f97014a9d357f9fc7a1fb9bef626db979f4bcc40707f1ff4d1b3ab5bfa4014b5b90183cce87bb88672082c9f042679129642fe33c7cfe88bd63867c56cc2f66ded6eb72d0dfe745d5627ff5db0222c43c2922101d7a3f80195d238660d6e38b66639ba8be9f3b1849c3cca10fdfaa5dd35e482d02ed240351e99b197642b03a417a986860ecc9e5a23a5d9be2642b9fe02d592f6edc1fda2511c5f0c735da6b178715cbba7ffe100e20e418c9dac880f21abfb7f7d582d805b7c627112f410279106e9ae078a3574cde48aa257d3b79be17bde5383a066a3f540205c6cc518049cf3ddec1761b3d160857cf16a1d179d35d3d767d556e6678ce5a16d38f58a7d96e7562768402c0eaa12d48c816571c24dc096cc0a53047589694374168f542a876dc48ec17f9778fa2dc415d325fd47c9162f9518b9af522d628cc0df288110b53327dd353eeab6ca17097d0e66a01654ef145cd60486e73542e25e4d547da855f256c2b1f33b3d29644b2f399a2b72e4f50a92f607bc68f1472453e3e48e78e9f6223ada22264ced49a9b089e39f223dfab9340d91c3e25dbe1f79be2041397d599019da86b8dde49ab2204c58814bda33a6041592caa2e665e9a59942905eef4567113d14ee1a75f593f311c58a2748e9e0863c54776eb61568a26bc31384ec33667aeecf1e250d1d01fca1061f08922f2367bde020865b0d37b823cbd3bfb41e4a255896cb5db35e5432b9e66fdfb0f8cdebf611495237b338d3c403843460af1a7c9dfa95807f6722549564c816e756bbf6c4452b015e5281e6b778753456ad271e17e5ba196972c1910c9681b8a7a20c942546d4b74d26d40059a630b1f257685025a24d9e06d227c5735fbf238147f4c3c49216f7fb6a6d58d7144da4203fa16eac8033d67d159a2b403f89aed0af63a09f0abf6fa42c00ffbba550469ee3522cf73214a2162e4a498176a56dcb7b33915c4c5467b3365e1ab12b3eaf22d79b995e9d53260086aa61c5b828df0a7c192a12654703d2571503a742b677b633a1601c8d162e468ece480a94381b6f263061f803784ca8b731975a2e00a693826b908ffb61c35dc4f14904f6f6d4008446d1541053d39a97ed972073a5884a263aa3bc96872d8b1257ba9e8f1f491791f00ea244ccb973cd377a23841e03721208d9f402e04f2eac1d495738d12751516e03a0fb22f92987ea40a80aca126a63f6d780432d2a820ee36377593af092008279249880b4b6bd7038b8ca8c1a99655f2b8b5263ea54a0d1eeb073de70532e9d03cd715e89a5e7a0a7c0736d6b1fb79c953c970ca517d5e6a2df4b1da1cde8c85d70a183b177cc22485b1c7a323c57dda3824dc748a50ca1b86c36ec391066ef5edc256cea9f83249a75e5b0cf8375f740375922dce612166caf28d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff3203929101005a2d4e4f4d50212068747470733a2f2f6769746875622e636f6d2f6a6f7368756179616275742f7a2d6e6f6d70ffffffff0200b33f71000000001976a914d5292d1d67d66195ae146647ca8f8c61c5e2ae9888ac00e1f5050000000017a914a5edac498df98a87bbfb5eb228c51769127f8cbc8700000000" +} From cf49b737d3fc1b3c5f6f28150b4a6d667c42566b Mon Sep 17 00:00:00 2001 From: dax Date: Wed, 7 Mar 2018 01:59:06 +0100 Subject: [PATCH 36/79] Enable verbose mode in blockchain.transaction.get (#397) getrawtransaction: optional verbose mode Update PROTOCOL.rst docs --- .travis.yml | 1 + docs/PROTOCOL.rst | 34 ++++++++++--- server/controller.py | 5 +- server/daemon.py | 4 +- tests/server/test_api.py | 104 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 tests/server/test_api.py diff --git a/.travis.yml b/.travis.yml index a76ba3234..c34efed9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ install: - pip install pyrocksdb - pip install tribus-hash - pip install pytest-cov + - pip install pylru # command to run tests script: pytest --cov=server --cov=lib --cov=wallet # Dont report coverage from nightly diff --git a/docs/PROTOCOL.rst b/docs/PROTOCOL.rst index 173eba1e4..66e7fc803 100644 --- a/docs/PROTOCOL.rst +++ b/docs/PROTOCOL.rst @@ -930,11 +930,11 @@ Protocol Version 1.2 Protocol version 1.2 introduces new methods `blockchain.block.headers`, `mempool.get_fee_histogram`. -`blockchain.block.get_chunk` and all methods beginning - `blockchain.address.` are deprecated and support will be removed in - some future protocol version. You should update your code to use - `blockchain.block.headers` and `Script Hashes`_ with the scripthash - methods introduced in protocol 1.1 instead. +* `blockchain.block.get_chunk` and all methods beginning `blockchain.address.` are deprecated + and support will be removed in some future protocol version. You should update your code to use `blockchain.block.headers` + and `Script Hashes`_ with the scripthash methods introduced in protocol 1.1 instead. + +* `blockchain.transaction.get` now has an optional parameter *verbose*. blockchain.block.headers ======================== @@ -978,7 +978,7 @@ Return concatenated block headers as hexadecimal from the main chain. { "count": 2, - "hex": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299'" + "hex": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299" "max": 2016 } @@ -1003,3 +1003,25 @@ intervals is currently not part of the protocol. .. _JSON RPC 1.0: http://json-rpc.org/wiki/specification .. _JSON RPC 2.0: http://json-rpc.org/specification + + +blockchain.transaction.get +========================== + +Return a raw transaction. + + blockchain.transaction.get(**tx_hash**, **verbose**=False) + + **tx_hash** + + The transaction hash as a hexadecimal string. + + **verbose** + Verbose mode, passed on to bitcoind in **getrawtransaction**. + +**Response** + + In non-verbose mode return the raw transaction as a hexadecimal string. + + If verbose, returns what your coin's bitcoind returns; see its + documentation for details. diff --git a/server/controller.py b/server/controller.py index 454de2b3e..b77a22815 100644 --- a/server/controller.py +++ b/server/controller.py @@ -842,13 +842,14 @@ async def relayfee(self): to the daemon's memory pool.''' return await self.daemon_request('relayfee') - async def transaction_get(self, tx_hash): + async def transaction_get(self, tx_hash, verbose=False): '''Return the serialized raw transaction given its hash tx_hash: the transaction hash as a hexadecimal string + verbose: passed on to the daemon ''' self.assert_tx_hash(tx_hash) - return await self.daemon_request('getrawtransaction', tx_hash) + return await self.daemon_request('getrawtransaction', tx_hash, verbose) async def transaction_get_1_0(self, tx_hash, height=None): '''Return the serialized raw transaction given its hash diff --git a/server/daemon.py b/server/daemon.py index d5554e8ec..bd70a3cd5 100644 --- a/server/daemon.py +++ b/server/daemon.py @@ -262,9 +262,9 @@ async def relayfee(self): network_info = await self.getnetworkinfo() return network_info['relayfee'] - async def getrawtransaction(self, hex_hash): + async def getrawtransaction(self, hex_hash, verbose=False): '''Return the serialized raw transaction with the given hash.''' - return await self._send_single('getrawtransaction', (hex_hash, 0)) + return await self._send_single('getrawtransaction', (hex_hash, int(verbose))) async def getrawtransactions(self, hex_hashes, replace_errs=True): '''Return the serialized raw transactions with the given hashes. diff --git a/tests/server/test_api.py b/tests/server/test_api.py new file mode 100644 index 000000000..4d52629b3 --- /dev/null +++ b/tests/server/test_api.py @@ -0,0 +1,104 @@ +import asyncio +from unittest import mock + +from lib.jsonrpc import RPCError +from server.env import Env +from server.controller import Controller + +loop = asyncio.get_event_loop() + + +def set_env(): + env = mock.create_autospec(Env) + env.coin = mock.Mock() + env.loop_policy = None + env.max_sessions = 0 + env.max_subs = 0 + env.max_send = 0 + env.bandwidth_limit = 0 + env.identities = '' + env.tor_proxy_host = env.tor_proxy_port = None + env.peer_discovery = env.PD_SELF = False + env.daemon_url = 'http://localhost:8000/' + return env + + +async def coro(res): + return res + + +def raise_exception(exc, msg): + raise exc(msg) + + +def ensure_text_exception(test, exception): + res = err = None + try: + res = loop.run_until_complete(test) + except Exception as e: + err = e + assert isinstance(err, exception), (res, err) + + +def test_transaction_get(): + async def test_verbose_ignore_by_backend(): + env = set_env() + sut = Controller(env) + sut.daemon_request = mock.Mock() + sut.daemon_request.return_value = coro('11'*32) + res = await sut.transaction_get('ff'*32, True) + assert res == '11'*32 + + async def test_verbose_ok(): + env = set_env() + sut = Controller(env) + sut.daemon_request = mock.Mock() + response = { + "hex": "00"*32, + "blockhash": "ff"*32 + } + sut.daemon_request.return_value = coro(response) + res = await sut.transaction_get('ff'*32, True) + assert res == response + + response = { + "hex": "00"*32, + "blockhash": None + } + sut.daemon_request.return_value = coro(response) + res = await sut.transaction_get('ff'*32, True) + assert res == response + + async def test_no_verbose(): + env = set_env() + sut = Controller(env) + sut.daemon_request = mock.Mock() + response = 'cafebabe'*64 + sut.daemon_request.return_value = coro(response) + res = await sut.transaction_get('ff'*32) + assert res == response + + async def test_verbose_failure(): + env = set_env() + sut = Controller(env) + sut.daemon_request = mock.Mock() + sut.daemon_request.return_value = coro(raise_exception(RPCError, 'some unhandled error')) + await sut.transaction_get('ff' * 32, True) + + async def test_wrong_txhash(): + env = set_env() + sut = Controller(env) + sut.daemon_request = mock.Mock() + await sut.transaction_get('cafe') + sut.daemon_request.assert_not_called() + + loop.run_until_complete(asyncio.gather( + *[ + test_verbose_ignore_by_backend(), + test_verbose_ok(), + test_no_verbose() + ] + )) + + for error_test in [test_verbose_failure, test_wrong_txhash]: + ensure_text_exception(error_test(), RPCError) From 28531bb452301bb7ac17e36c5bcc4efd738dc296 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 7 Mar 2018 10:30:40 +0800 Subject: [PATCH 37/79] Initial attempt at Sphinx documentation --- docs/Makefile | 20 ++++ docs/conf.py | 155 +++++++++++++++++++++++++++++ docs/index.rst | 21 ++++ docs/make.bat | 36 +++++++ docs/mempool_fee_get_histogram.rst | 15 +++ docs/protocol-new.rst | 132 ++++++++++++++++++++++++ 6 files changed, 379 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/mempool_fee_get_histogram.rst create mode 100644 docs/protocol-new.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..ca0957f05 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = ElectrumX +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..dbacfb2a5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'ElectrumX' +copyright = '2018, Neil Booth' +author = 'Neil Booth' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '1.2.1' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'ElectrumXdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'ElectrumX.tex', 'ElectrumX Documentation', + 'Neil Booth', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'electrumx', 'ElectrumX Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'ElectrumX', 'ElectrumX Documentation', + author, 'ElectrumX', 'One line description of project.', + 'Miscellaneous'), +] \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..2d117c5e3 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,21 @@ +.. ElectrumX documentation master file, created by + sphinx-quickstart on Mon Mar 5 22:39:16 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to ElectrumX's documentation! +===================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + protocol-new + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..b42ca1bfb --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=ElectrumX + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/mempool_fee_get_histogram.rst b/docs/mempool_fee_get_histogram.rst new file mode 100644 index 000000000..7debce542 --- /dev/null +++ b/docs/mempool_fee_get_histogram.rst @@ -0,0 +1,15 @@ +.. method:: mempool.get_fee_histogram() + + Return a histogram of the fee rates paid by transactions in the + memory pool, weighted by transaction size. + + **Response** + + The histogram is an array of [*fee*, *vsize*] pairs, where *vsize_n* is + the cumulative virtual size of mempool transactions with a fee rate + in the interval [*fee*_(n-1), *fee*_n)], and *fee*_(n-1) > *fee*_n. + + Fee intervals may have variable size. The choice of appropriate + intervals is currently not part of the protocol. + + .. versionadded:: 1.2 diff --git a/docs/protocol-new.rst b/docs/protocol-new.rst new file mode 100644 index 000000000..f50f5689b --- /dev/null +++ b/docs/protocol-new.rst @@ -0,0 +1,132 @@ +Electrum Protocol +================= + +This is intended to be a reference for client and server authors +alike. + + +Message Stream +-------------- + +Clients and servers communicate using **JSON RPC** over an unspecified +underlying stream transport protocol, typically TCP or SSL. + +`JSON RPC 1.0 ` and `JSON RPC +2.0 ` are specified; use of +version 2.0 is encouraged but not required. Server support of batch +requests is encouraged for version 1.0 but not required. Clients +making batch requests should limit their size depending on the nature +of their query, because servers will limit response size as an +anti-DoS mechanism. + +Eeach RPC call, and each response, is separated by a single newline in +their respective streams. The JSON specification does not permit +control characters within strings, so no confusion is possible there. +However it does permit newlines as extraneous whitespace between +elements; client and server MUST NOT use newlines in such a way. + +If using JSON RPC 2.0's feature of parameter passing by name, the +names shown in the description of the method in question MUST be used. + +A server advertising support for a particular protocol version MUST +support each method documented for that protocol version, unless the +method is explicitly marked optional. It may support other methods or +additional parameters with unspecified behaviour. Use of additional +parameters is discouraged as it may conflict with future versions of +the protocol. + + +Notifications +------------- + +Some RPC calls are subscriptions, which, after the initial response, +will send a :dfn:`notification` each time the thing subscribed to +changes. The `method` of the notification is the same as the method +of the subscription, and the `params` of the notification (and their +names) are given in the documentation of the method. + + +Protocol Negotiation +-------------------- + +It is desirable to have a way to enhance and improve the protocol +without forcing servers and clients to upgrade at the same time. +Protocol negotiation is not implemented in any client or server at +present to the best of my knowledge, so care is needed to ensure +current clients and servers continue to operate as expected. + +Protocol versions are denoted by dotted "a.b" strings, where *m* is +the major version number and *n* the minor version number. For +example: "1.5". + +A party to a connection will speak all protocol versions in a range, +say from `protocol_min` to `protocol_max`, which may be the same. +When a connection is made, both client and server must initially +assume the protocol to use is their own `protocol_min`. + +The client should send a :func:`server.version` RPC call as early as +possible in order to negotiate the precise protocol version; see its +description for more detail. All responses received in the stream +from and including the server's response to this call will use the +negotiated protocol version. + + +Script Hashes +------------- + +A :defn:`script hash` is the hash of the binary bytes of the locking +script (ScriptPubKey), expressed as a hexadecimal string. The hash +function to use is given by the "hash_function" member of +:func:`server.features` (currently "sha256" only). Like for block and +transaction hashes, when converting the big-endian binary hash to a +hexadecimal string the least-significant byte appears first, and the +most-significant byte last. + +For example, the legacy Bitcoin address from the genesis block:: + + 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa + +has P2PKH script:: + + 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac + +with SHA256 hash:: + + 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b + +which is sent to the server reversed as:: + + 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 + +By subscribing to this hash you can find P2PKH payments to that address. + +One public key for that address (the genesis block public key) is:: + + 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f + +which has P2PK script:: + + 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac + +with SHA256 hash:: + + 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 + +which is sent to the server reversed as:: + + 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 + +By subscribing to this hash you can find P2PK payments to that public +key. + +.. note:: The Genesis block coinbase is unspendable and therefore not +indexed. It will not show with the above P2PK script hash subscription. + + +.. toctree:: + :maxdepth: 1 + :caption: Methods: + + mempool_fee_get_histogram From fe37b470e5c7a651f790a4839af606cf4454faf8 Mon Sep 17 00:00:00 2001 From: Neil Booth Date: Wed, 7 Mar 2018 13:14:45 +0800 Subject: [PATCH 38/79] More work on sphinx docs --- docs/HOWTO.rst | 16 +- docs/index.rst | 63 ++++++- docs/mempool_fee_get_histogram.rst | 15 -- docs/protocol-basics.rst | 125 ++++++++++++++ docs/protocol-methods.rst | 268 +++++++++++++++++++++++++++++ docs/protocol-new.rst | 125 +------------- 6 files changed, 467 insertions(+), 145 deletions(-) delete mode 100644 docs/mempool_fee_get_histogram.rst create mode 100644 docs/protocol-basics.rst create mode 100644 docs/protocol-methods.rst diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 8bfc7f7af..3faa58763 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -1,4 +1,9 @@ -============= +.. _HOWTO: + +===== +HOWTO +===== + Prerequisites ============= @@ -13,16 +18,17 @@ Package Notes Python3 ElectrumX uses asyncio. Python version >= 3.6 is **required**. `aiohttp`_ Python library for asynchronous HTTP. Version >= - 1.0 required; I am using 1.0.5. + 1.0 required; I am using 3.0.1. `pylru`_ Python LRU cache package. I'm using 1.0.9. DB Engine I use `plyvel`_ 0.9, a Python interface to LevelDB. A database engine package is required but others are supported (see **Database Engine** below). -`x11_hash`_ Only required for DASH. Python X11 Hash package. Only - required if for Dash. Version 1.4 tested. ================ ======================== -You need to be running a non-pruning bitcoin daemon with:: +Some coins need an additional package, typically for their block hash +functions. For example, `x11_hash`_ is required for DASH. + +You **must** to be running a non-pruning bitcoin daemon with:: txindex=1 diff --git a/docs/index.rst b/docs/index.rst index 2d117c5e3..4564cf60f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,14 +3,73 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to ElectrumX's documentation! -===================================== +.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master + :target: https://travis-ci.org/kyuupichan/electrumx +.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg + :target: https://coveralls.io/github/kyuupichan/electrumx + +========= +ElectrumX +========= + +A reimplementation of Electrum-Server for a future with bigger blocks. + + :Licence: MIT + :Language: Python (>= 3.6) + :Author: Neil Booth + + +Getting Started +=============== + +See :ref:`HOWTO`. + +There is also an `installer`_ available that simplifies the +installation on various Linux-based distributions, and a `Dockerfile`_ +available . + +.. _installer: https://github.com/bauerj/electrumx-installer +.. _Dockerfile: https://github.com/lukechilds/docker-electrumx + + +Features +======== + +- Efficient, lightweight reimplementation of electrum-server +- Fast synchronization of bitcoin mainnet from Genesis. Recent + hardware should synchronize in well under 24 hours. The fastest + time to height 448k (mid January 2017) reported is under 4h 30m. On + the same hardware JElectrum would take around 4 days and + electrum-server probably around 1 month. +- The full current Electrum protocol is implemented. +- Various configurable means of controlling resource consumption and + handling denial of service attacks. These include maximum + connection counts, subscription limits per-connection and across all + connections, maximum response size, per-session bandwidth limits, + and session timeouts. +- Minimal resource usage once caught up and serving clients; tracking the + transaction mempool appears to be the most expensive part. +- Fully asynchronous processing of new blocks, mempool updates, and + client requests. Busy clients should not noticeably impede other + clients' requests and notifications, nor the processing of incoming + blocks and mempool updates. +- Daemon failover. More than one daemon can be specified, and + ElectrumX will failover round-robin style if the current one fails + for any reason. +- Peer discovery protocol removes need for IRC +- Coin abstraction makes compatible altcoin and testnet support easy. + .. toctree:: :maxdepth: 2 :caption: Contents: + ENVIRONMENT + HOWTO + PEER_DISCOVERY + RPC-INTERFACE protocol-new + ARCHITECTURE Indices and tables diff --git a/docs/mempool_fee_get_histogram.rst b/docs/mempool_fee_get_histogram.rst deleted file mode 100644 index 7debce542..000000000 --- a/docs/mempool_fee_get_histogram.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. method:: mempool.get_fee_histogram() - - Return a histogram of the fee rates paid by transactions in the - memory pool, weighted by transaction size. - - **Response** - - The histogram is an array of [*fee*, *vsize*] pairs, where *vsize_n* is - the cumulative virtual size of mempool transactions with a fee rate - in the interval [*fee*_(n-1), *fee*_n)], and *fee*_(n-1) > *fee*_n. - - Fee intervals may have variable size. The choice of appropriate - intervals is currently not part of the protocol. - - .. versionadded:: 1.2 diff --git a/docs/protocol-basics.rst b/docs/protocol-basics.rst new file mode 100644 index 000000000..aee447e84 --- /dev/null +++ b/docs/protocol-basics.rst @@ -0,0 +1,125 @@ +Protocol Basics +=============== + +Message Stream +-------------- + +Clients and servers communicate using **JSON RPC** over an unspecified +underlying stream transport protocol, typically TCP or SSL. + +Two standards `JSON RPC 1.0 +`_ and `JSON RPC 2.0 +`_ are specified; use of version +2.0 is encouraged but not required. Server support of batch requests +is encouraged for version 1.0 but not required. Clients making batch +requests should limit their size depending on the nature of their +query, because servers will limit response size as an anti-DoS +mechanism. + +Eeach RPC call, and each response, is separated by a single newline in +their respective streams. The JSON specification does not permit +control characters within strings, so no confusion is possible there. +However it does permit newlines as extraneous whitespace between +elements; client and server MUST NOT use newlines in such a way. + +If using JSON RPC 2.0's feature of parameter passing by name, the +names shown in the description of the method in question MUST be used. + +A server advertising support for a particular protocol version MUST +support each method documented for that protocol version, unless the +method is explicitly marked optional. It may support other methods or +additional parameters with unspecified behaviour. Use of additional +parameters is discouraged as it may conflict with future versions of +the protocol. + + +Notifications +------------- + +Some RPC calls are subscriptions, which, after the initial response, +will send a JSON RPC :dfn:`notification` each time the thing +subscribed to changes. The `method` of the notification is the same +as the method of the subscription, and the `params` of the +notification (and their names) are given in the documentation of the +method. + + +Version Negotiation +------------------- + +It is desirable to have a way to enhance and improve the protocol +without forcing servers and clients to upgrade at the same time. +Protocol negotiation is not implemented in any client or server at +present to the best of my knowledge, so care is needed to ensure +current clients and servers continue to operate as expected. + +Protocol versions are denoted by dotted "a.b" strings, where *m* is +the major version number and *n* the minor version number. For +example: "1.5". + +A party to a connection will speak all protocol versions in a range, +say from `protocol_min` to `protocol_max`, which may be the same. +When a connection is made, both client and server must initially +assume the protocol to use is their own `protocol_min`. + +The client should send a :func:`server.version` RPC call as early as +possible in order to negotiate the precise protocol version; see its +description for more detail. All responses received in the stream +from and including the server's response to this call will use the +negotiated protocol version. + +.. _script hashes: + +Script Hashes +------------- + +A :dfn:`script hash` is the hash of the binary bytes of the locking +script (ScriptPubKey), expressed as a hexadecimal string. The hash +function to use is given by the "hash_function" member of +:func:`server.features` (currently "sha256" only). Like for block and +transaction hashes, when converting the big-endian binary hash to a +hexadecimal string the least-significant byte appears first, and the +most-significant byte last. + +For example, the legacy Bitcoin address from the genesis block:: + + 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa + +has P2PKH script:: + + 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac + +with SHA256 hash:: + + 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b + +which is sent to the server reversed as:: + + 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 + +By subscribing to this hash you can find P2PKH payments to that address. + +One public key for that address (the genesis block public key) is:: + + 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f + +which has P2PK script:: + + 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb + 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac + +with SHA256 hash:: + + 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 + +which is sent to the server reversed as:: + + 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 + +By subscribing to this hash you can find P2PK payments to that public +key. + +.. note:: The Genesis block coinbase is unspendable and therefore not + indexed. It will not show with the above P2PK script hash + subscription. diff --git a/docs/protocol-methods.rst b/docs/protocol-methods.rst new file mode 100644 index 000000000..412179941 --- /dev/null +++ b/docs/protocol-methods.rst @@ -0,0 +1,268 @@ +Protocol Methods +================ + +mempool.get_fee_histogram +------------------------- + + Return a histogram of the fee rates paid by transactions in the + memory pool, weighted by transaction size. + +**Signature** + + .. function:: mempool.get_fee_histogram() + + .. versionadded:: 1.2 + +**Result** + + The histogram is an array of [*fee*, *vsize*] pairs, where |vsize_n| + is the cumulative virtual size of mempool transactions with a fee rate + in the interval [|fee_n1|, |fee_n|], and |fee_n1| > |fee_n|. + + .. |vsize_n| replace:: vsize\ :sub:`n` + .. |fee_n| replace:: fee\ :sub:`n` + .. |fee_n1| replace:: fee\ :sub:`n-1` + + Fee intervals may have variable size. The choice of appropriate + intervals is currently not part of the protocol. + +**Example Result** + + :: + + [[12, 128812], [4, 92524], [2, 6478638], [1, 22890421]] + + +server.add_peer +--------------- + + This call is intended for a new server to get itself into a server's + peers list, and should not be used by wallet clients. + +**Signature** + + .. function:: server.add_peer(features) + + .. versionadded:: 1.1 + + * **features** + + The same information that a call to the sender's + :func:`server.features` RPC call would return. + +**Result** + + A boolean indicating whether the request was tentatively accepted. + The requesting server will appear in :func:`server.peers.subscribe` + when further sanity checks complete successfully. + + +server.banner +------------- + + Return a banner to be shown in the Electrum console. + +**Signature** + + .. function:: server.banner() + +**Result** + + A string. + +**Example Result** + + :: + + "Welcome to Electrum!" + + +server.donation_address +----------------------- + + Return a server donation address. + +**Signature** + + .. function:: server.donation_address() + +**Result** + + A string. + +**Example Result** + + :: + + "1BWwXJH3q6PRsizBkSGm2Uw4Sz1urZ5sCj" + + +server.features +--------------- + + Return a list of features and services supported by the server. + +**Signature** + + .. function:: server.features() + +**Result** + + A dictionary of keys and values. Each key represents a feature or + service of the server, and the value gives additional information. + + The following features MUST be reported by the server. Additional + key-value pairs may be returned. + + * **hosts** + + A dictionary, keyed by host name, that this server can be reached + at. Normally this will only have a single entry; other entries + can be used in case there are other connection routes (e.g. Tor). + + The value for a host is itself a dictionary, with the following + optional keys: + + * **ssl_port** + + An integer. Omit or set to **null** if SSL connectivity is not + provided. + + * **tcp_port** + + An integer. Omit or set to **null** if TCP connectivity is not + provided. + + A server should ignore information provided about any host other + than the one it connected to. + + * **genesis_hash** + + The hash of the genesis block. This is used to detect if a peer + is connected to one serving a different network. + + * **hash_function** + + The hash function the server uses for :ref:`script hashing +