From 90ef0fda09876db473bcd33cad5a422976227efe Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Thu, 2 Oct 2025 11:25:23 -0700 Subject: [PATCH 1/3] Adding prometheus compose to track system resources during netowrk test - added 2 targets to Makefile to control prometheus/cadvisor --- Makefile | 8 ++++ docker/docker-compose.prom.yml | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 docker/docker-compose.prom.yml diff --git a/Makefile b/Makefile index c705fe1..22bc6fa 100644 --- a/Makefile +++ b/Makefile @@ -105,6 +105,14 @@ genesis: check-not-running | build $(CHAINSTATE_DIR) /usr/bin/sudo # Secondary name to boot the genesis network up-genesis: genesis +up-prom: + @echo "Starting $(PROJECT) prometheus monitoring" + docker compose -f docker/docker-compose.prom.yml --profile default -p $(PROJECT) up -d + +down-prom: + @echo "Shutting down $(PROJECT) prometheus monitoring" + docker compose -f docker/docker-compose.prom.yml --profile default -p $(PROJECT) down + # Shut down the network (chainstate and logs will be preserved) down: backup-logs current-chainstate-dir @echo "Shutting down $(PROJECT) network" diff --git a/docker/docker-compose.prom.yml b/docker/docker-compose.prom.yml new file mode 100644 index 0000000..f62e9d0 --- /dev/null +++ b/docker/docker-compose.prom.yml @@ -0,0 +1,74 @@ +x-common-vars: + - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks} + - &DOCKER_NETWORK_CIDR 10.0.0.0/24 + - &DOCKER_NETWORK_GATEWAY 10.0.0.1 + - &PROMETHEUS_IP 10.0.0.252 + - &CADVISOR_IP 10.0.0.253 + +name: monitoring + +configs: + # prometheus scraping config + prometheus.yml: + content: | + # Global config + global: + external_labels: + monitor: "hacknet" + # A scrape configuration containing Prometheus and Docker + scrape_configs: + - job_name: cadvisor + scrape_interval: 5s + static_configs: + - targets: + - cadvisor:8080 + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: '^(go_|cadvisor_version_info).+' + action: drop + - job_name: stacks + scrape_interval: 5s + static_configs: + - targets: + - stacks-miner-1:9153 + - stacks-miner-2:9153 + - stacks-miner-3:9153 +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + networks: + default: + ipv4_address: *PROMETHEUS_IP + configs: + - source: prometheus.yml + target: /etc/prometheus/prometheus.yml + depends_on: + - cadvisor + ports: + - '0.0.0.0:${PROM_HTTP:-9090}:${PROM_HTTP:-9090}' + command: + - --config.file=/etc/prometheus/prometheus.yml + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + networks: + default: + ipv4_address: *CADVISOR_IP + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - '0.0.0.0:8080:8080' + command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory" + +networks: + default: + name: *DOCKER_NETWORK + ipam: + driver: default + config: + - subnet: *DOCKER_NETWORK_CIDR + gateway: *DOCKER_NETWORK_GATEWAY From 471dd36b8866b2ed090f073259cdc30e74163b7b Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Thu, 23 Oct 2025 10:54:45 -0700 Subject: [PATCH 2/3] Update Makefile and Readme for the (up/down)-prom targets --- Makefile | 4 ++-- README.md | 34 ++++++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 22bc6fa..67d9773 100644 --- a/Makefile +++ b/Makefile @@ -107,11 +107,11 @@ up-genesis: genesis up-prom: @echo "Starting $(PROJECT) prometheus monitoring" - docker compose -f docker/docker-compose.prom.yml --profile default -p $(PROJECT) up -d + docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring up -d down-prom: @echo "Shutting down $(PROJECT) prometheus monitoring" - docker compose -f docker/docker-compose.prom.yml --profile default -p $(PROJECT) down + docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring down # Shut down the network (chainstate and logs will be preserved) down: backup-logs current-chainstate-dir diff --git a/README.md b/README.md index 2f8025d..2fc2b79 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ make log stacks-signer-1 -- -f make log stacks-signer-1 ``` +### Container management #### Pause/Unpause service To pause all services on the network ```sh @@ -88,6 +89,19 @@ Restart the stopped service make start ``` +#### Force stop the hacknet network +If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network. + +```sh +make down-force +``` + +Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*` +```sh +make clean +``` + +### Additional Features #### Stress the CPU To simulate CPU load. Can be modified with: - `STRESS_CORES` to target how many worker threads (default will use all cores) @@ -99,6 +113,12 @@ make stress STRESS_CORES=10 STRESS_TIMEOUT=60 make stress ``` +#### Monitor chain heights +Run a script outputting the current chain heights of each miner +```sh +make monitor +``` + #### Create a chainstate snapshot - Setting the env var `PAUSE_HEIGHT` is optional to pause the chain at a specific height, else a default of Bitcoin block `999999999999` is used. - Setting the env var `MINE_INTERVAL_EPOCH3` is recommended to reach the `PAUSE_HEIGHT` more quickly to create the snapshot @@ -129,19 +149,17 @@ ex: CHAINSTATE_ARCHIVE=./docker/chainstate_new.tar.zstd make up ``` -#### Force stop the hacknet network -If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network. - +#### Prometheus sidecar +##### Run prometheus and cadvisor +Runs a prometheus container to record data collected by `cadvisor` for tracking host/container metrics ```sh -make down-force +make up-prom ``` - -Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*` +##### Stop prometheus and cadvisor ```sh -make clean +make down-prom ``` - ## Containers - **bitcoin**: Runs a bitcoin regtest node From b397988288354dd1b1299f6e934c76d4c3564576 Mon Sep 17 00:00:00 2001 From: wileyj <2847772+wileyj@users.noreply.github.com> Date: Wed, 29 Oct 2025 10:25:39 -0700 Subject: [PATCH 3/3] Linting - Updating compose files based on linter output. - Updating workflow to check all compose files in ./docker/ - Updated dclintrc to ignore quotes for ports (default only passes with single quotes, not double quotes) --- .dclintrc | 1 + .github/workflows/linter.yml | 2 +- docker/docker-compose.prom.yml | 136 ++++++++++++++++----------------- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/.dclintrc b/.dclintrc index 0137ea8..fe31f34 100644 --- a/.dclintrc +++ b/.dclintrc @@ -3,5 +3,6 @@ rules: services-alphabetical-order: 0 service-image-require-explicit-tag: 0 + require-quotes-in-ports: 0 quiet: false debug: true diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index b844751..9b68f43 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: docker-compose-linter/dclint-github-action@18659f6a7956706cb67cf9c1ad5e55f4352cbc17 # 1.6.0 with: - path: ./docker/docker-compose.yml + path: ./docker/ recursive: true diff --git a/docker/docker-compose.prom.yml b/docker/docker-compose.prom.yml index f62e9d0..2523d3d 100644 --- a/docker/docker-compose.prom.yml +++ b/docker/docker-compose.prom.yml @@ -1,74 +1,74 @@ x-common-vars: - - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks} - - &DOCKER_NETWORK_CIDR 10.0.0.0/24 - - &DOCKER_NETWORK_GATEWAY 10.0.0.1 - - &PROMETHEUS_IP 10.0.0.252 - - &CADVISOR_IP 10.0.0.253 + - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks} + - &DOCKER_NETWORK_CIDR 10.0.0.0/24 + - &DOCKER_NETWORK_GATEWAY 10.0.0.1 + - &PROMETHEUS_IP 10.0.0.252 + - &CADVISOR_IP 10.0.0.253 name: monitoring - -configs: - # prometheus scraping config - prometheus.yml: - content: | - # Global config - global: - external_labels: - monitor: "hacknet" - # A scrape configuration containing Prometheus and Docker - scrape_configs: - - job_name: cadvisor - scrape_interval: 5s - static_configs: - - targets: - - cadvisor:8080 - metric_relabel_configs: - - source_labels: [ __name__ ] - regex: '^(go_|cadvisor_version_info).+' - action: drop - - job_name: stacks - scrape_interval: 5s - static_configs: - - targets: - - stacks-miner-1:9153 - - stacks-miner-2:9153 - - stacks-miner-3:9153 services: - prometheus: - image: prom/prometheus:latest - container_name: prometheus - networks: - default: - ipv4_address: *PROMETHEUS_IP - configs: - - source: prometheus.yml - target: /etc/prometheus/prometheus.yml - depends_on: - - cadvisor - ports: - - '0.0.0.0:${PROM_HTTP:-9090}:${PROM_HTTP:-9090}' - command: - - --config.file=/etc/prometheus/prometheus.yml - cadvisor: - image: gcr.io/cadvisor/cadvisor:latest - container_name: cadvisor - networks: - default: - ipv4_address: *CADVISOR_IP - volumes: - - /:/rootfs:ro - - /var/run:/var/run:rw - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - ports: - - '0.0.0.0:8080:8080' - command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory" + prometheus: + image: prom/prometheus:latest + container_name: prometheus + depends_on: + - cadvisor + configs: + - source: prometheus.yml + target: /etc/prometheus/prometheus.yml + ports: + - "0.0.0.0:9090:9090" + networks: + default: + ipv4_address: *PROMETHEUS_IP + command: + - --config.file=/etc/prometheus/prometheus.yml + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + ports: + - "0.0.0.0:8080:8080" + networks: + default: + ipv4_address: *CADVISOR_IP + command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory" networks: - default: - name: *DOCKER_NETWORK - ipam: - driver: default - config: - - subnet: *DOCKER_NETWORK_CIDR - gateway: *DOCKER_NETWORK_GATEWAY + default: + name: *DOCKER_NETWORK + ipam: + driver: default + config: + - subnet: *DOCKER_NETWORK_CIDR + gateway: *DOCKER_NETWORK_GATEWAY + +configs: + # prometheus scraping config + prometheus.yml: + content: | + # Global config + global: + external_labels: + monitor: "hacknet" + # A scrape configuration containing Prometheus and Docker + scrape_configs: + - job_name: cadvisor + scrape_interval: 5s + static_configs: + - targets: + - cadvisor:8080 + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: '^(go_|cadvisor_version_info).+' + action: drop + - job_name: stacks + scrape_interval: 5s + static_configs: + - targets: + - stacks-miner-1:9153 + - stacks-miner-2:9153 + - stacks-miner-3:9153