Chore/prometheus

.dclintrc

@@ -3,5 +3,6 @@
 rules: 
     services-alphabetical-order: 0
     service-image-require-explicit-tag: 0
+    require-quotes-in-ports: 0
 quiet: false
 debug: true

.github/workflows/linter.yml

@@ -11,5 +11,5 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: docker-compose-linter/dclint-github-action@18659f6a7956706cb67cf9c1ad5e55f4352cbc17 # 1.6.0
         with:
-          path: ./docker/docker-compose.yml
+          path: ./docker/
           recursive: true

Makefile

@@ -105,6 +105,14 @@ genesis: check-not-running | build $(CHAINSTATE_DIR) /usr/bin/sudo
 # Secondary name to boot the genesis network
 up-genesis: genesis
 
+up-prom:
+	@echo "Starting $(PROJECT) prometheus monitoring"
+	docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring up -d
+
+down-prom:
+	@echo "Shutting down $(PROJECT) prometheus monitoring"
+	docker compose -f docker/docker-compose.prom.yml --profile default -p monitoring down
+
 # Shut down the network (chainstate and logs will be preserved)
 down: backup-logs current-chainstate-dir
 	@echo "Shutting down $(PROJECT) network"

README.md

@@ -58,6 +58,7 @@ make log stacks-signer-1 -- -f
 make log stacks-signer-1
 ```
 
+### Container management
 #### Pause/Unpause service
 To pause all services on the network
 ```sh
@@ -88,6 +89,19 @@ Restart the stopped service
 make start <service name>
 ```
 
+#### Force stop the hacknet network
+If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network.
+
+```sh
+make down-force
+```
+
+Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*`
+```sh
+make clean
+```
+
+### Additional Features
 #### Stress the CPU
 To simulate CPU load. Can be modified with:
 - `STRESS_CORES` to target how many worker threads (default will use all cores)
@@ -99,6 +113,12 @@ make stress
 STRESS_CORES=10 STRESS_TIMEOUT=60 make stress
 ```
 
+#### Monitor chain heights
+Run a script outputting the current chain heights of each miner
+```sh
+make monitor
+```
+
 #### Create a chainstate snapshot
 - Setting the env var `PAUSE_HEIGHT` is optional to pause the chain at a specific height, else a default of Bitcoin block `999999999999` is used.
 - Setting the env var `MINE_INTERVAL_EPOCH3` is recommended to reach the `PAUSE_HEIGHT` more quickly to create the snapshot
@@ -129,19 +149,17 @@ ex:
 CHAINSTATE_ARCHIVE=./docker/chainstate_new.tar.zstd make up
 ```
 
-#### Force stop the hacknet network
-If the network is in a "stuck" state where the Makefile targets are not stopping the services (i.e. the `.current-chainstate-dir` file was removed while network was running), `down-force` may be used to force stop the network.
-
+#### Prometheus sidecar
+##### Run prometheus and cadvisor
+Runs a prometheus container to record data collected by `cadvisor` for tracking host/container metrics
 ```sh
-make down-force
+make up-prom
 ```
-
-Additionally, `clean` target will call `down-force` *and also* delete any chainstates on disk in `./docker/chainstate/*`
+##### Stop prometheus and cadvisor
 ```sh
-make clean
+make down-prom
 ```
 
-
 ## Containers
 
 - **bitcoin**: Runs a bitcoin regtest node

docker/docker-compose.prom.yml

@@ -0,0 +1,74 @@
+x-common-vars:
+    - &DOCKER_NETWORK ${DOCKER_NETWORK:-stacks}
+    - &DOCKER_NETWORK_CIDR 10.0.0.0/24
+    - &DOCKER_NETWORK_GATEWAY 10.0.0.1
+    - &PROMETHEUS_IP 10.0.0.252
+    - &CADVISOR_IP 10.0.0.253
+
+name: monitoring
+services:
+    prometheus:
+        image: prom/prometheus:latest
+        container_name: prometheus
+        depends_on:
+            - cadvisor
+        configs:
+            - source: prometheus.yml
+              target: /etc/prometheus/prometheus.yml
+        ports:
+            - "0.0.0.0:9090:9090"
+        networks:
+            default:
+                ipv4_address: *PROMETHEUS_IP
+        command:
+            - --config.file=/etc/prometheus/prometheus.yml
+    cadvisor:
+        image: gcr.io/cadvisor/cadvisor:latest
+        container_name: cadvisor
+        volumes:
+            - /:/rootfs:ro
+            - /var/run:/var/run:rw
+            - /sys:/sys:ro
+            - /var/lib/docker/:/var/lib/docker:ro
+        ports:
+            - "0.0.0.0:8080:8080"
+        networks:
+            default:
+                ipv4_address: *CADVISOR_IP
+        command: "--enable_metrics=cpu,cpuLoad,disk,diskIO,memory"
+
+networks:
+    default:
+        name: *DOCKER_NETWORK
+        ipam:
+            driver: default
+            config:
+                - subnet: *DOCKER_NETWORK_CIDR
+                  gateway: *DOCKER_NETWORK_GATEWAY
+
+configs:
+    # prometheus scraping config
+    prometheus.yml:
+        content: |
+            # Global config
+            global:
+              external_labels:
+                monitor: "hacknet"
+            # A scrape configuration containing Prometheus and Docker
+            scrape_configs:
+            - job_name: cadvisor
+              scrape_interval: 5s
+              static_configs:
+                - targets:
+                  - cadvisor:8080
+              metric_relabel_configs:
+                - source_labels: [ __name__ ]
+                  regex: '^(go_|cadvisor_version_info).+'
+                  action: drop
+            - job_name: stacks
+              scrape_interval: 5s
+              static_configs:
+                - targets:
+                  - stacks-miner-1:9153
+                  - stacks-miner-2:9153
+                  - stacks-miner-3:9153