diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6564446..40d4ddb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ repos: -- repo: https://github.com/psf/black - rev: 22.3.0 + - repo: https://github.com/psf/black + rev: 25.1.0 hooks: - id: black - language_version: python3 + name: black + entry: black ./src/embit diff --git a/README.md b/README.md index 5237a85..331c32f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Should remain minimal to fit in a microcontroller. Also easy to audit. Examples can be found in [`examples/`](./examples) folder. -Documentation: https://embit.rocks/ +Documentation: Support the project: `bc1qd4flfrxjctls9ya244u39hd67pcprhvka723gv` @@ -28,9 +28,10 @@ PyPi installation includes prebuilt libraries for common platforms (win, macos, If you want to build the lib yourself, see: [Building secp256k1 for `embit`](/secp256k1/README.md). - ## Using non-English BIP39 wordlists + [BIP39](https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md) defines wordlists for: + * English * Japanese * Korean @@ -45,6 +46,7 @@ If you want to build the lib yourself, see: [Building secp256k1 for `embit`](/se `embit` assumes English and does not include the other wordlists in order to keep this as slim as possible. However, you can override this default by providing an alternate wordlist to any of the mnemonic-handling methods: + ``` spanish_wordlist = [ "ábaco", @@ -66,22 +68,67 @@ mnemonic_to_bytes(mnemonic, wordlist=spanish_wordlist) mnemonic_from_bytes(bytes_data, wordlist=spanish_wordlist) ``` - # Development +It's worth to setup a virtual environment to not mess with your system. To do it, install `virtualenv`: + +```bash +pipx install virtualenv +``` + +create a virtual environment: + +```bash +virtualenv +``` + +and activate it: + +```bash +chmod +x +source /bin/activate +``` + +## Dependencies + Install in developer mode with dev dependencies: ```sh pip install -e .[dev] ``` -Install pre-commit hook: +If you're using `zsh`, try: + +```bash +pip install -e '.[dev]' +``` + +Also you should have `bitcoind` and `elementsd` (liquid) installed + +## Pre-commit tools + +Before commit your changes, perform some `pre-commit` check (formatting, linting and test). +First assert that you have `pre-commit` dependencies installed: ```sh pre-commit install ``` -Run tests with desktop python: +Then run: + +```bash +pre-commit run --all-files +``` + +## Tests + +Our tests are divided in unit tests and integration tests. The unit tests is for check if anything in the code +is running properly. The integration tests are for check if those changes are compliant with bitcoin-core and +elements-core. + +### Unit tests + +If you prefer, you can run only the unit tests: ```sh pytest @@ -93,3 +140,29 @@ Run tests with micropython: cd tests micropython ./run_tests.py ``` + +### Integration Tests + +Integration tests need a properly setup to run. This means that we should setup +our system with bitcoind, elementsd and berklay-db. We have a script called `./tests/run.sh` +that build, from latests releases, the `bitcoind` and `elementsd` and put then on a temporary directory, +so it will not mess with your system. + +But also means that you should have some dependencies installed in your system: + +* gcc +* make +* cmake +* pkg-config +* libboost +* berkley-db + +Once you have those tools installed, you can run: + +```bash +# This will download and install both bitcoind and elementsd, +# as well configure them to our needs as well setup some +# environment variables to make a proper reproductible environment. +# Then it will run both unit tests and integration tests. +./tests/run.sh +``` diff --git a/pyproject.toml b/pyproject.toml index fb8fcfe..da82e00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,11 @@ warn_unreachable = true warn_unused_configs = true no_implicit_reexport = false + +[tool.black] +exclude = ''' +/( + \.git + | \.venv +)/ +''' diff --git a/src/embit/base.py b/src/embit/base.py index 9dbac73..7eb2147 100644 --- a/src/embit/base.py +++ b/src/embit/base.py @@ -1,4 +1,5 @@ """Base classes""" + from io import BytesIO from binascii import hexlify, unhexlify diff --git a/src/embit/compact.py b/src/embit/compact.py index 0138f39..4a62523 100644 --- a/src/embit/compact.py +++ b/src/embit/compact.py @@ -1,4 +1,5 @@ -""" Compact Int parsing / serialization """ +"""Compact Int parsing / serialization""" + import io diff --git a/src/embit/descriptor/miniscript.py b/src/embit/descriptor/miniscript.py index 9783afc..61d9774 100644 --- a/src/embit/descriptor/miniscript.py +++ b/src/embit/descriptor/miniscript.py @@ -901,7 +901,7 @@ def inner_compile(self): def __len__(self): return len(self.arg) + 1 - + def verify(self): super().verify() if self.arg.type != "V": diff --git a/src/embit/liquid/blip32.py b/src/embit/liquid/blip32.py index 1d49471..9048acb 100644 --- a/src/embit/liquid/blip32.py +++ b/src/embit/liquid/blip32.py @@ -1,4 +1,5 @@ """BIP-32 for blinding keys. Non-standard yet!!!""" + import sys from .. import bip32, ec from .networks import NETWORKS diff --git a/src/embit/liquid/networks.py b/src/embit/liquid/networks.py index 2fa0837..a78d003 100644 --- a/src/embit/liquid/networks.py +++ b/src/embit/liquid/networks.py @@ -33,8 +33,8 @@ def get_network(name): }, "elementsregtest": { "name": "Liquid Regtest", - "wif": b"\xEF", - "p2pkh": b"\x6F", + "wif": b"\xef", + "p2pkh": b"\x6f", "p2sh": b"\x4b", "bp2sh": b"\x04\x4b", "bech32": "ert", @@ -54,7 +54,7 @@ def get_network(name): # config: https://liquidtestnet.com/ "liquidtestnet": { "name": "Liquid Testnet", - "wif": b"\xEF", + "wif": b"\xef", "p2pkh": b"\x24", "p2sh": b"\x13", "bp2sh": b"\x17\x13", diff --git a/src/embit/liquid/pset.py b/src/embit/liquid/pset.py index 3a2c8b8..09faa3a 100644 --- a/src/embit/liquid/pset.py +++ b/src/embit/liquid/pset.py @@ -349,10 +349,12 @@ def blinded_vout(self): self.value_commitment or self.value, self.script_pubkey, self.ecdh_pubkey, - None - if not self.surjection_proof - else TxOutWitness( - Proof(self.surjection_proof), RangeProof(self.range_proof) + ( + None + if not self.surjection_proof + else TxOutWitness( + Proof(self.surjection_proof), RangeProof(self.range_proof) + ) ), ) diff --git a/src/embit/misc.py b/src/embit/misc.py index fc2c804..f3c950e 100644 --- a/src/embit/misc.py +++ b/src/embit/misc.py @@ -1,4 +1,5 @@ """Misc utility functions used across embit""" + import sys # implementation-specific functions and libraries: diff --git a/src/embit/networks.py b/src/embit/networks.py index 6f1a541..ea3c6ba 100644 --- a/src/embit/networks.py +++ b/src/embit/networks.py @@ -21,9 +21,9 @@ }, "test": { "name": "Testnet", - "wif": b"\xEF", - "p2pkh": b"\x6F", - "p2sh": b"\xC4", + "wif": b"\xef", + "p2pkh": b"\x6f", + "p2sh": b"\xc4", "bech32": "tb", "xprv": b"\x04\x35\x83\x94", "xpub": b"\x04\x35\x87\xcf", @@ -39,9 +39,9 @@ }, "regtest": { "name": "Regtest", - "wif": b"\xEF", - "p2pkh": b"\x6F", - "p2sh": b"\xC4", + "wif": b"\xef", + "p2pkh": b"\x6f", + "p2sh": b"\xc4", "bech32": "bcrt", "xprv": b"\x04\x35\x83\x94", "xpub": b"\x04\x35\x87\xcf", @@ -57,9 +57,9 @@ }, "signet": { "name": "Signet", - "wif": b"\xEF", - "p2pkh": b"\x6F", - "p2sh": b"\xC4", + "wif": b"\xef", + "p2pkh": b"\x6f", + "p2sh": b"\xc4", "bech32": "tb", "xprv": b"\x04\x35\x83\x94", "xpub": b"\x04\x35\x87\xcf", diff --git a/src/embit/psbtview.py b/src/embit/psbtview.py index 4bcd664..8d513da 100644 --- a/src/embit/psbtview.py +++ b/src/embit/psbtview.py @@ -13,6 +13,7 @@ Makes sense to run gc.collect() after processing of each scope to free memory. """ + # TODO: refactor, a lot of code is duplicated here from transaction.py from collections import OrderedDict import hashlib @@ -339,7 +340,7 @@ def vin(self, i, compress=None): vout = int.from_bytes(v, "little") self.seek_to_scope(i) - v = self.get_value(b"\x10", from_current=True) or b"\xFF\xFF\xFF\xFF" + v = self.get_value(b"\x10", from_current=True) or b"\xff\xff\xff\xff" sequence = int.from_bytes(v, "little") return TransactionInput(txid, vout, sequence=sequence) diff --git a/src/embit/util/key.py b/src/embit/util/key.py index 13b01d9..9d97812 100644 --- a/src/embit/util/key.py +++ b/src/embit/util/key.py @@ -2,6 +2,7 @@ Copy-paste from key.py in bitcoin test_framework. This is a fallback option if the library can't do ctypes bindings to secp256k1 library. """ + import random import hmac import hashlib diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/tests/test_psbt.py b/tests/integration/tests/test_psbt.py index b425722..1508366 100644 --- a/tests/integration/tests/test_psbt.py +++ b/tests/integration/tests/test_psbt.py @@ -1,5 +1,5 @@ from unittest import TestCase, skip -from util.bitcoin import daemon +from ..util.bitcoin import daemon import random from embit.descriptor import Descriptor from embit.descriptor.checksum import add_checksum diff --git a/tests/integration/tests/test_pset.py b/tests/integration/tests/test_pset.py index 2f976d2..3899e3c 100644 --- a/tests/integration/tests/test_pset.py +++ b/tests/integration/tests/test_pset.py @@ -1,5 +1,5 @@ from unittest import TestCase, skip -from util.liquid import daemon +from ..util.liquid import daemon import random import time import os diff --git a/tests/integration/util/bitcoin.py b/tests/integration/util/bitcoin.py index 0d3552c..9040d27 100644 --- a/tests/integration/util/bitcoin.py +++ b/tests/integration/util/bitcoin.py @@ -1,20 +1,28 @@ import subprocess import os +import sys import time import signal import shutil + from .rpc import BitcoinRPC +try: + EMBIT_TEMP_DIR = os.getenv("EMBIT_TEMP_DIR") +except Exception as e: + print(e) + sys.exit(1) + class Bitcoind: - datadir = os.path.abspath("./chain/bitcoin") + datadir = os.path.join(EMBIT_TEMP_DIR, "data", "bitcoin", "chain") rpcport = 18778 port = 18779 rpcuser = "bitcoin" rpcpassword = "secret" name = "Bitcoin Core" retry_count = 10 - binary = "bitcoind" + binary = os.path.join(EMBIT_TEMP_DIR, "binaries", "bitcoind") def __init__(self): self._rpc = None @@ -49,9 +57,7 @@ def start(self): os.makedirs(self.datadir) except: pass - self.proc = subprocess.Popen( - self.cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid - ) + self.proc = subprocess.Popen(self.cmd.split(" ")) time.sleep(1) self.get_coins() @@ -67,17 +73,17 @@ def mine(self, n=1): def stop(self): print(f"Shutting down {self.name}") + self.rpc.stop() os.killpg( os.getpgid(self.proc.pid), signal.SIGTERM ) # Send the signal to all the process groups - time.sleep(3) for i in range(self.retry_count): try: # shutil.rmtree(self.datadir) return except Exception as e: print(f"Exception: {e}") - print(f"Retrying in 1 second... {i}/{retry_count}") + print(f"Retrying in 1 second... {i}/{self.retry_count}") time.sleep(1) diff --git a/tests/integration/util/liquid.py b/tests/integration/util/liquid.py index fada302..9370898 100644 --- a/tests/integration/util/liquid.py +++ b/tests/integration/util/liquid.py @@ -1,15 +1,22 @@ import os +import sys from .bitcoin import Bitcoind +try: + EMBIT_TEMP_DIR = os.getenv("EMBIT_TEMP_DIR") +except Exception as e: + print(e) + sys.exit(1) + class Elementsd(Bitcoind): - datadir = os.path.abspath("./chain/elements") + datadir = os.path.join(EMBIT_TEMP_DIR, "data", "elements") rpcport = 18998 port = 18999 rpcuser = "liquid" rpcpassword = "secret" name = "Elements Core" - binary = "elementsd" + binary = os.path.join(EMBIT_TEMP_DIR, "binaries", "elementsd") @property def cmd(self): diff --git a/tests/integration/util/rpc.py b/tests/integration/util/rpc.py index 9ebfafe..e049f8f 100644 --- a/tests/integration/util/rpc.py +++ b/tests/integration/util/rpc.py @@ -360,6 +360,8 @@ def multi(self, calls: list, **kwargs): url = self.url if "wallet" in kwargs: url = url + "/wallet/{}".format(kwargs["wallet"]) + + print(f"POST {url}: {payload}") r = self.session.post( url, data=json.dumps(payload), headers=headers, timeout=timeout ) @@ -373,10 +375,11 @@ def multi(self, calls: list, **kwargs): def __getattr__(self, method): def fn(*args, **kwargs): - r = self.multi([(method, *args)], **kwargs)[0] - if r["error"] is not None: - raise RpcError("Request error: %s" % r["error"]["message"], r) - return r["result"] + res = self.multi([(method, *args)], **kwargs)[0] + if "error" in res and res["error"] is not None: + print(res) + raise RpcError("Request error: %s" % res["error"]["message"], res) + return res["result"] return fn diff --git a/tests/run.sh b/tests/run.sh new file mode 100755 index 0000000..9d80212 --- /dev/null +++ b/tests/run.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash + +# Prepare a directory with the right testing binaries to avoid conflicts with different versions of embit. +# +# What this script do ? +# +# 0. While checking build dependencies; +# +# 1. Clone and build elementsd at $EMBIT_TEMP_DIR/binaries/elementsd. +# +# 2. Build bitcoind at $EMBIT_TEMP_DIR/binaries/bitcoind. +# +# 3. export EMBIT_TEMP_DIR which points to /tmp/embit-integration-tests.${HEAD_COMMIT_HASH}/binaries + +set -e + +PROJECT_DIR=$(pwd) + +# We expect for the current dir to be the root dir of the project. +EMBIT_PROJ_DIR=$(git rev-parse --show-toplevel) + +# This helps us to keep track of the actual version being tested without conflicting with any already installed binaries. +GIT_DESCRIBE=$(git describe --tags --always) + +export EMBIT_TEMP_DIR="/tmp/embit-integration-tests.${GIT_DESCRIBE}" + +# Dont use mktemp so we can have deterministic results for each version of embit. +mkdir -p $EMBIT_TEMP_DIR/binaries + +check_installed() { + if ! command -v "$1" &>/dev/null; then + echo "You must have $1 installed to run those tests!" + exit 1 + fi +} + +# From https://github.com/murrayn/bitcoin/blob/22e9afe40d987f4f90bc8469f9475df138fe6261/contrib/install_db4.sh +sha256_check() { + # Args: + # + if check_installed sha256sum; then + echo "${1} ${2}" | sha256sum + elif check_installed sha256; then + if [ "$(uname)" = "FreeBSD" ]; then + sha256 -c "${1}" "${2}" + else + echo "${1} ${2}" | sha256 -c + fi + else + echo "${1} ${2}" | shasum -a 256 -c + fi +} + +# From https://github.com/murrayn/bitcoin/blob/22e9afe40d987f4f90bc8469f9475df138fe6261/contrib/install_db4.sh +http_get() { + # Args: + # + # It's acceptable that we don't require SSL here because we manually verify + # content hashes below. + # + if [ -f "${2}" ]; then + echo "File ${2} already exists; not downloading again" + elif check_installed curl; then + curl -L "${1}" -o "${2}" + elif check_installed wget; then + wget --no-check-certificate "${1}" -O "${2}" + else + echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." + exit 1 + fi + + sha256_check "${3}" "${2}" +} + +# From https://github.com/vinteumorg/Floresta/blob/master/tests/prepare.sh +build_core() { + # Download and build bitcoind only + mkdir -p "$EMBIT_TEMP_DIR/binaries/build" + cd "$EMBIT_TEMP_DIR/binaries/build" || exit 1 + + echo "Downloading and Building Bitcoin Core..." + git clone https://github.com/bitcoin/bitcoin + cd bitcoin || exit 1 + + # If BITCOIN_REVISION is set, check it out + # if not, set the default to 29 (the last one) + bitcoin_rev="${BITCOIN_REVISION:-29.0}" + if [ -n "$bitcoin_rev" ]; then + # Check if the revision exists as a tag only + if git --no-pager tag -l | grep -q "^v$bitcoin_rev$"; then + git checkout "v$bitcoin_rev" + else + echo "bitcoin 'v$bitcoin_rev' is not a valid tag." + exit 1 + fi + fi + + # Check compatibility with cmake arguments with those used with make + # See https://gist.github.com/hebasto/2ef97d3a726bfce08ded9df07f7dab5e and + # https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Autotools-to-CMake-Options-Mapping + major_version="${bitcoin_rev%%.*}" + if [ "$major_version" -ge 29 ]; then + cmake -S . -B build \ + -DBUILD_CLI=OFF \ + -DBUILD_TESTS=OFF \ + -DENABLE_WALLET=ON \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DENABLE_EXTERNAL_SIGNER=OFF \ + -DINSTALL_MAN=OFF + cmake --build build --target bitcoind -j"$(nproc)" + mv $EMBIT_TEMP_DIR/binaries/build/bitcoin/build/bin/bitcoind $EMBIT_TEMP_DIR/binaries/bitcoind + else + ./autogen.sh + ./configure \ + --without-gui \ + --disable-tests \ + --disable-bench + make -j$(nproc) + mv $EMBIT_TEMP_DIR/binaries/build/bitcoin/src/bitcoind $EMBIT_TEMP_DIR/binaries/bitcoind + fi + + rm -rf $EMBIT_TEMP_DIR/binaries/build +} + +build_elements() { + + # Download and build elements + mkdir -p "$EMBIT_TEMP_DIR/binaries/build" + cd "$EMBIT_TEMP_DIR/binaries/build" + + echo "Downloading and Building Elements Core..." + git clone https://github.com/ElementsProject/elements + cd elements + + # If ELEMENTS_REVISION is set, check it out + elements_rev="${ELEMENTS_REVISION:-23.3.0}" + if [ -n "$elements_rev" ]; then + # Check if the revision exists as a tag only + if git --no-pager tag -l | grep -q "^elements-$elements_rev$"; then + git checkout "elements-$elements_rev" + else + echo "elements 'elements-$elements_rev' is not a valid tag." + exit 1 + fi + fi + + # will need + # export LDFLAGS="-L/opt/homebrew/opt/berkeley-db@4/lib" + # export CPPFLAGS="-I/opt/homebrew/opt/berkeley-db@4/include" + ./autogen.sh + ./configure \ + --disable-tests \ + --disable-bench + make -j$(nproc) + mv $EMBIT_TEMP_DIR/binaries/build/elements/src/elementsd $EMBIT_TEMP_DIR/binaries/elementsd + rm -rf $EMBIT_TEMP_DIR/binaries/build +} + +check_installed git +check_installed gcc +check_installed make +check_installed cmake +check_installed autoconf + +# Check if bitcoind is already built or if --build is passed +if [ ! -f $EMBIT_TEMP_DIR/binaries/bitcoind ] || [ "$1" == "--build" ]; then + build_core +else + echo "Bitcoind already built, skipping..." +fi + +# Check if bitcoind is already built or if --build is passed +if [ ! -f $EMBIT_TEMP_DIR/binaries/elementsd ] || [ "$1" == "--build" ]; then + build_elements +else + echo "Elementsd already built, skipping..." +fi + +echo "Binaries available in temporary directory at $EMBIT_TEMP_DIR/binaries" + +# From # From https://github.com/vinteumorg/Floresta/blob/master/tests/run.sh +# Clean existing data/logs directories before running the tests +echo "Cleaning previous data at $EMBIT_TEMP_DIR/data" +rm -rf "$EMBIT_TEMP_DIR/data" + +# Detect if --preserve-data-dir is among args +# and forward args to uv +PRESERVE_DATA=false + +for arg in "$@"; do + if [[ "$arg" == "--preserve-data-dir" ]]; then + PRESERVE_DATA=true + fi +done + +cd $PROJECT_DIR +pip install -e '.[dev]' + +# Run the re-freshed tests +echo "Running unittests with 'python $(pwd)/tests/run_tests.py'" +python $(pwd)/tests/run_tests.py || exit 1 + +echo "Running integration with 'python $(pwd)/tests/integration/run_tests.py'" +python $(pwd)/tests/integration/run_tests.py || exit 1 + +echo "Tests passed" +# Clean up the data dir if we succeeded and --preserve-data-dir was not passed +if [ $? -eq 0 ] && [ "$PRESERVE_DATA" = false ]; then + echo "Cleaning up the data dir at $EMBIT_TEMP_DIR" + rm -rf $EMBIT_TEMP_DIR/data +for ((i = 0; i < 10; i++)); do + echo "$i" +done + +exit 0