test: improve dev environment for integration tests

.pre-commit-config.yaml

@@ -1,6 +1,7 @@
 repos:
--   repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
     hooks:
     - id: black
-      language_version: python3
+      name: black
+      entry: black ./src/embit 

README.md

@@ -6,7 +6,7 @@ Should remain minimal to fit in a microcontroller. Also easy to audit.
 
 Examples can be found in [`examples/`](./examples) folder.
 
-Documentation: https://embit.rocks/
+Documentation: <https://embit.rocks/>
 
 Support the project: `bc1qd4flfrxjctls9ya244u39hd67pcprhvka723gv`
 
@@ -28,9 +28,10 @@ PyPi installation includes prebuilt libraries for common platforms (win, macos,
 
 If you want to build the lib yourself, see: [Building secp256k1 for `embit`](/secp256k1/README.md).
 
-
 ## Using non-English BIP39 wordlists
+
 [BIP39](https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md) defines wordlists for:
+
 * English
 * Japanese
 * Korean
@@ -45,6 +46,7 @@ If you want to build the lib yourself, see: [Building secp256k1 for `embit`](/se
 `embit` assumes English and does not include the other wordlists in order to keep this as slim as possible.
 
 However, you can override this default by providing an alternate wordlist to any of the mnemonic-handling methods:
+
 ```
 spanish_wordlist = [
     "ábaco",
@@ -66,22 +68,67 @@ mnemonic_to_bytes(mnemonic, wordlist=spanish_wordlist)
 mnemonic_from_bytes(bytes_data, wordlist=spanish_wordlist)
 ```
 
-
 # Development
 
+It's worth to setup a virtual environment to not mess with your system. To do it, install `virtualenv`:
+
+```bash
+pipx install virtualenv
+```
+
+create a virtual environment:
+
+```bash
+virtualenv <path-to-embit-virtualenv>
+```
+
+and activate it:
+
+```bash
+chmod +x <path-to-embit-virtualenv>
+source <path-to-embit-virtualenv>/bin/activate
+```
+
+## Dependencies
+
 Install in developer mode with dev dependencies:
 
 ```sh
 pip install -e .[dev]
 ```
 
-Install pre-commit hook:
+If you're using `zsh`, try:
+
+```bash
+pip install -e '.[dev]'
+```
+
+Also you should have `bitcoind` and `elementsd` (liquid) installed
+
+## Pre-commit tools
+
+Before commit your changes, perform some `pre-commit` check (formatting, linting and test).
+First assert that you have `pre-commit` dependencies installed:
 
 ```sh
 pre-commit install
 ```
 
-Run tests with desktop python:
+Then run:
+
+```bash
+pre-commit run --all-files
+```
+
+## Tests
+
+Our tests are divided in unit tests and integration tests. The unit tests is for check if anything in the code
+is running properly. The integration tests are for check if those changes are compliant with bitcoin-core and
+elements-core.
+
+### Unit tests
+
+If you prefer, you can run only the unit tests:
 
 ```sh
 pytest
@@ -93,3 +140,29 @@ Run tests with micropython:
 cd tests
 micropython ./run_tests.py
 ```
+
+### Integration Tests
+
+Integration tests need a properly setup to run. This means that we should setup
+our system with bitcoind, elementsd and berklay-db. We have a script called `./tests/run.sh`
+that build, from latests releases, the `bitcoind` and `elementsd` and put then on a temporary directory,
+so it will not mess with your system.
+
+But also means that you should have some dependencies installed in your system:
+
+* gcc
+* make
+* cmake
+* pkg-config
+* libboost
+* berkley-db
+
+Once you have those tools installed, you can run:
+
+```bash
+# This will download and install both bitcoind and elementsd,
+# as well configure them to our needs as well setup some 
+# environment variables to make a proper reproductible environment.
+# Then it will run both unit tests and integration tests.
+./tests/run.sh
+```

pyproject.toml

@@ -62,3 +62,11 @@ warn_unreachable = true
 warn_unused_configs = true
 no_implicit_reexport = false
 
+
+[tool.black]
+exclude = '''
+/(
+    \.git
+  | \.venv
+)/
+'''

src/embit/base.py

@@ -1,4 +1,5 @@
 """Base classes"""
+
 from io import BytesIO
 from binascii import hexlify, unhexlify
 

src/embit/compact.py

@@ -1,4 +1,5 @@
-""" Compact Int parsing / serialization """
+"""Compact Int parsing / serialization"""
+
 import io
 
 

src/embit/descriptor/miniscript.py

@@ -901,7 +901,7 @@ def inner_compile(self):
 
     def __len__(self):
         return len(self.arg) + 1
-    
+
     def verify(self):
         super().verify()
         if self.arg.type != "V":

src/embit/liquid/blip32.py

@@ -1,4 +1,5 @@
 """BIP-32 for blinding keys. Non-standard yet!!!"""
+
 import sys
 from .. import bip32, ec
 from .networks import NETWORKS

src/embit/liquid/networks.py

@@ -33,8 +33,8 @@ def get_network(name):
     },
     "elementsregtest": {
         "name": "Liquid Regtest",
-        "wif": b"\xEF",
-        "p2pkh": b"\x6F",
+        "wif": b"\xef",
+        "p2pkh": b"\x6f",
         "p2sh": b"\x4b",
         "bp2sh": b"\x04\x4b",
         "bech32": "ert",
@@ -54,7 +54,7 @@ def get_network(name):
     # config: https://liquidtestnet.com/
     "liquidtestnet": {
         "name": "Liquid Testnet",
-        "wif": b"\xEF",
+        "wif": b"\xef",
         "p2pkh": b"\x24",
         "p2sh": b"\x13",
         "bp2sh": b"\x17\x13",

src/embit/liquid/pset.py

@@ -349,10 +349,12 @@ def blinded_vout(self):
             self.value_commitment or self.value,
             self.script_pubkey,
             self.ecdh_pubkey,
-            None
-            if not self.surjection_proof
-            else TxOutWitness(
-                Proof(self.surjection_proof), RangeProof(self.range_proof)
+            (
+                None
+                if not self.surjection_proof
+                else TxOutWitness(
+                    Proof(self.surjection_proof), RangeProof(self.range_proof)
+                )
             ),
         )
 

src/embit/misc.py

@@ -1,4 +1,5 @@
 """Misc utility functions used across embit"""
+
 import sys
 
 # implementation-specific functions and libraries:

src/embit/networks.py

@@ -21,9 +21,9 @@
     },
     "test": {
         "name": "Testnet",
-        "wif": b"\xEF",
-        "p2pkh": b"\x6F",
-        "p2sh": b"\xC4",
+        "wif": b"\xef",
+        "p2pkh": b"\x6f",
+        "p2sh": b"\xc4",
         "bech32": "tb",
         "xprv": b"\x04\x35\x83\x94",
         "xpub": b"\x04\x35\x87\xcf",
@@ -39,9 +39,9 @@
     },
     "regtest": {
         "name": "Regtest",
-        "wif": b"\xEF",
-        "p2pkh": b"\x6F",
-        "p2sh": b"\xC4",
+        "wif": b"\xef",
+        "p2pkh": b"\x6f",
+        "p2sh": b"\xc4",
         "bech32": "bcrt",
         "xprv": b"\x04\x35\x83\x94",
         "xpub": b"\x04\x35\x87\xcf",
@@ -57,9 +57,9 @@
     },
     "signet": {
         "name": "Signet",
-        "wif": b"\xEF",
-        "p2pkh": b"\x6F",
-        "p2sh": b"\xC4",
+        "wif": b"\xef",
+        "p2pkh": b"\x6f",
+        "p2sh": b"\xc4",
         "bech32": "tb",
         "xprv": b"\x04\x35\x83\x94",
         "xpub": b"\x04\x35\x87\xcf",

src/embit/psbtview.py

@@ -13,6 +13,7 @@
 
 Makes sense to run gc.collect() after processing of each scope to free memory.
 """
+
 # TODO: refactor, a lot of code is duplicated here from transaction.py
 from collections import OrderedDict
 import hashlib
@@ -339,7 +340,7 @@ def vin(self, i, compress=None):
         vout = int.from_bytes(v, "little")
 
         self.seek_to_scope(i)
-        v = self.get_value(b"\x10", from_current=True) or b"\xFF\xFF\xFF\xFF"
+        v = self.get_value(b"\x10", from_current=True) or b"\xff\xff\xff\xff"
         sequence = int.from_bytes(v, "little")
 
         return TransactionInput(txid, vout, sequence=sequence)

src/embit/util/key.py

@@ -2,6 +2,7 @@
 Copy-paste from key.py in bitcoin test_framework.
 This is a fallback option if the library can't do ctypes bindings to secp256k1 library.
 """
+
 import random
 import hmac
 import hashlib

tests/integration/tests/test_psbt.py

@@ -1,5 +1,5 @@
 from unittest import TestCase, skip
-from util.bitcoin import daemon
+from ..util.bitcoin import daemon
 import random
 from embit.descriptor import Descriptor
 from embit.descriptor.checksum import add_checksum

tests/integration/tests/test_pset.py

@@ -1,5 +1,5 @@
 from unittest import TestCase, skip
-from util.liquid import daemon
+from ..util.liquid import daemon
 import random
 import time
 import os

tests/integration/util/bitcoin.py

@@ -1,20 +1,28 @@
 import subprocess
 import os
+import sys
 import time
 import signal
 import shutil
+
 from .rpc import BitcoinRPC
 
+try:
+    EMBIT_TEMP_DIR = os.getenv("EMBIT_TEMP_DIR")
+except Exception as e:
+    print(e)
+    sys.exit(1)
+
 
 class Bitcoind:
-    datadir = os.path.abspath("./chain/bitcoin")
+    datadir = os.path.join(EMBIT_TEMP_DIR, "data", "bitcoin", "chain")
     rpcport = 18778
     port = 18779
     rpcuser = "bitcoin"
     rpcpassword = "secret"
     name = "Bitcoin Core"
     retry_count = 10
-    binary = "bitcoind"
+    binary = os.path.join(EMBIT_TEMP_DIR, "binaries", "bitcoind")
 
     def __init__(self):
         self._rpc = None
@@ -49,9 +57,7 @@ def start(self):
             os.makedirs(self.datadir)
         except:
             pass
-        self.proc = subprocess.Popen(
-            self.cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid
-        )
+        self.proc = subprocess.Popen(self.cmd.split(" "))
         time.sleep(1)
         self.get_coins()
 
@@ -67,17 +73,17 @@ def mine(self, n=1):
 
     def stop(self):
         print(f"Shutting down {self.name}")
+        self.rpc.stop()
         os.killpg(
             os.getpgid(self.proc.pid), signal.SIGTERM
         )  # Send the signal to all the process groups
-        time.sleep(3)
         for i in range(self.retry_count):
             try:
                 # shutil.rmtree(self.datadir)
                 return
             except Exception as e:
                 print(f"Exception: {e}")
-                print(f"Retrying in 1 second... {i}/{retry_count}")
+                print(f"Retrying in 1 second... {i}/{self.retry_count}")
                 time.sleep(1)
 
 

tests/integration/util/liquid.py

@@ -1,15 +1,22 @@
 import os
+import sys
 from .bitcoin import Bitcoind
 
+try:
+    EMBIT_TEMP_DIR = os.getenv("EMBIT_TEMP_DIR")
+except Exception as e:
+    print(e)
+    sys.exit(1)
+
 
 class Elementsd(Bitcoind):
-    datadir = os.path.abspath("./chain/elements")
+    datadir = os.path.join(EMBIT_TEMP_DIR, "data", "elements")
     rpcport = 18998
     port = 18999
     rpcuser = "liquid"
     rpcpassword = "secret"
     name = "Elements Core"
-    binary = "elementsd"
+    binary = os.path.join(EMBIT_TEMP_DIR, "binaries", "elementsd")
 
     @property
     def cmd(self):