From 597c383a14a209d025e1dc45671b397c3414694d Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Nov 2025 08:26:38 +0700 Subject: [PATCH 1/8] add copilot slop --- docker/.dockerignore | 23 +++ docker/Dockerfile | 27 ++++ docker/README.md | 54 +++++++ docker/docker.md | 354 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 458 insertions(+) create mode 100644 docker/.dockerignore create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker.md diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 000000000..591bfe816 --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,23 @@ +# Ignore common build artifacts and large nix store paths +result +result.tar.gz +result-*.tar.gz +dist +dist-newstyle +dist/* +*.o +*.hi +.stack-work +.ghc.environment.* +.cabal-sandbox +.cabal-sandbox/* +node_modules +.git +*.lock +**/.DS_Store + +# Ignore Nix store references and temporary files +/nix +*.drv +*.nar + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..7e1f2cace --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,27 @@ +# Lightweight Dockerfile for a static cardano-db-sync binary +# +# Usage (example): +# 1. Build a static executable with Nix: +# nix build .#cardano-db-sync-linux +# 2. Copy the executable into this folder (follow symlinks): +# cp -L result/bin/cardano-db-sync docker/ +# 3. Build the image: +# docker build -t cardano-db-sync:lightweight docker/ +# +# Important: this Dockerfile assumes the binary is a fully static executable +# (statically linked). If it's not static, use a small base image (see docs). + +FROM scratch + +# Add the static executable produced by Nix. The builder step places +# the binary next to this Dockerfile before running `docker build`. +COPY cardano-db-sync /usr/local/bin/cardano-db-sync + +# Minimal entrypoint; configuration should be supplied via mounted files +# or environment variables. We avoid baked-in configs to keep images small +# and easier to customize. +ENTRYPOINT ["/usr/local/bin/cardano-db-sync"] + +# Default to printing help if no args supplied (preserves normal CLI behaviour) +CMD ["--help"] + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..08132ea2d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,54 @@ +Lightweight Docker image for cardano-db-sync +========================================== + +This folder contains a minimal Docker context and `Dockerfile` intended for use with +a statically-built `cardano-db-sync` executable produced by Nix. + +Why this exists +--------------- + +Building the full NixOS-based Docker image is a reproducible approach, but can lead +to large images and bundled configuration that make customization more difficult. +This lightweight approach lets users build a small, standard Docker image and supply +configuration at runtime via volumes or create their own derived images. + +Quickstart +---------- + +1. Build a static `cardano-db-sync` binary with Nix: + + ```bash + nix build .#cardano-db-sync-linux + ``` + +2. Copy the binary to this directory (follow symlinks): + + ```bash + cp -L result/bin/cardano-db-sync docker/ + ``` + +3. Build the image: + + ```bash + docker build -t cardano-db-sync:lightweight docker/ + ``` + +4. Run the container and mount your configuration (recommended): + + ```bash + docker run -v $PWD/environments/mainnet:/config \ + --env POSTGRES_HOST=postgres --env POSTGRES_PORT=5432 \ + cardano-db-sync:lightweight run --config /config/db-sync-config.json + ``` + +Notes +----- + +- The `Dockerfile` uses `FROM scratch` and therefore requires the binary to be + statically linked. If you need dynamic linking, change the base image to a small + distro such as `ubuntu:24.04` or `debian:bookworm-slim`. +- The image intentionally omits network configuration files. Use volumes or a + derived image if you want to embed configs. + +See also: the project's `doc/docker.md` for more details and guidance. + diff --git a/docker/docker.md b/docker/docker.md new file mode 100644 index 000000000..ade998529 --- /dev/null +++ b/docker/docker.md @@ -0,0 +1,354 @@ +# Docker + +## Quickstart + +We provide an example +[docker-compose.yml](https://github.com/IntersectMBO/cardano-db-sync/blob/master/docker-compose.example.yml) +to quickly get set up with `cardano-db-sync`. Keep in mind that this is only a template, +and users are encouraged to tweak it to meet their needs. + +It is not recommended to use this for a production set up, and should only be +used for local development. + +Create a working directory: + +```bash +mkdir ~/src +cd ~/src +``` + +Download the example Docker Compose file: + +```bash +curl -o docker-compose.yml \ + https://raw.githubusercontent.com/IntersectMBO/cardano-db-sync/master/docker-compose.example.yml +``` + +Start `cardano-node`, `postgresql`, and `cardano-db-sync` services: + +```bash +docker compose up -d && docker compose logs -f +``` + +The PostgreSQL database should now be exposed on localhost port `5432` + +To connect to PostgreSQL database: + +```bash +psql -h 0.0.0.0 -p 5432 -U postgres -d cexplorer +``` + +### Connecting to a Other Networks + +To connect to different network use `NETWORK` environment variable: + +```bash +NETWORK=preprod docker compose up -d && docker compose logs -f +``` + +## Running `cardano-db-sync` + +Start `cardano-db-sync`: + +```bash +docker run \ + --env NETWORK=mainnet \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + --volume db-sync-data:/var/lib/cexplorer \ + --volume node-ipc \ + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 +``` + +### Environment Variables + +#### `NETWORK` + +Specifies a network to connect to. If specified, `cardano-db-sync` will provide the +configuration files. If not specified, will call the `cardano-db-sync` executable as the +entrypoint. Possible values are: + + * mainnet + * preprod + * preview + * private + * sanchonet + * shelley_qa + +#### `POSTGRES_HOST` (required) + +The PostgreSQL server host to connect to. + +#### `POSTGRES_PORT` (required) + +Specifies the PostgreSQL server port to connect to. + +#### `POSTGRES_USER` (required) + +The PostgreSQL server user to connect as. + +#### `POSTGRES_PASSWORD` (required) + +Specifies the PostgreSQL server password to connect as. + +#### `RESTORE_SNAPSHOT` (optional) + +Specifies a `cardano-db-sync` snapshot archive to download and restore from. If omitted, +it will sync from genesis. see [Restoring From Snapshots](#restoring-from-snapshots) + +### `DB_SYNC_CONFIG` (optional) + +Overrides the `db-sync-config.json` provided by the network configuration. See [Overriding +Network Configuration](#overriding-network-configuration). + +#### `EXTRA_DB_SYNC_ARGS` (optional) + +Extra command line arguments to pass to `cardano-db-sync`. For example: + +```bash +docker run \ + --env NETWORK=mainnet \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + --env EXTRA_DB_SYNC_ARGS="--skip-fix" + --volume db-sync-data:/var/lib/cexplorer \ + --volume node-ipc \ + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 +``` + +### Overriding Network Configuration + +Overriding the configuration file can be done by passing the `DB_SYNC_CONFIG` environment +variable: + +```bash +docker run \ + --env NETWORK=mainnet \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + --env DB_SYNC_CONFIG=/config/db-sync-config.json \ + --volume db-sync-data:/var/lib/cexplorer \ + --volume node-ipc:/node-ipc \ + --volume $PWD/environments/mainnet:/config \ + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 +``` + +### Restoring From Snapshots + +Snapshots are provided for mainnet to restore the PostgreSQL database. See the [latest +releases](https://github.com/IntersectMBO/cardano-db-sync/releases) for a recent snapshot +matched with the `cardano-db-sync` version. + +Although, you can specify a URL directly, we recommend downloading the image separately to avoid +having to download it multiple times in case of failure. First, download the latest snapshot: + +```bash +curl -O https://update-cardano-mainnet.iohk.io/cardano-db-sync/13.3/db-sync-snapshot-schema-13.3-block-10611621-x86_64.tgz +``` + +Restore the snapshot with `RESTORE_SNAPSHOT`: + +```bash +docker run \ + --env NETWORK=mainnet \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + --env RESTORE_SNAPSHOT=/data/db-sync-snapshot-schema-13.3-block-10611621-x86_64.tgz \ + --volume db-sync-data:/var/lib/cexplorer \ + --volume node-ipc:/node-ipc \ + --volume $PWD:/data + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 +``` + +### Advanced Usage + +Excluding the `NETWORK` environment variable will simply call the `cardano-db-sync` +executable as the entrypoint, so you must pass command line arguments to projvide runtime +configuration. The `--schema-dir` argument is automatically set and is not required. + +```bash +docker run \ + --volume db-sync-data:/var/lib/cexplorer \ + --volume node-ipc:/node-ipc \ + --volume $PWD/environments/mainnet:/config \ + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 \ + run --config /config/db-sync-config.json --socket-path /node-ipc/node.socket +``` + +## Running `cardano-smash-server` + +Start `cardano-smash-server`: + +```bash +docker run \ + --env NETWORK=mainnet \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + --env SMASH_USER=smash-admin \ + --env SMASH_PASSWORD=smash-password \ + --publish 3100:3100 \ + --volume node-ipc:/node-ipc \ + ghcr.io/IntersectMBO/cardano-smash-server:13.3.0.0 +``` + +### Environment Variables + +#### `NETWORK` (optional) + +Specifies a network to connect to. If specified, `cardano-db-sync` will provide the +configuration files. If not specified, will call the `cardano-db-sync` executable as the +entrypoint. Possible values are: + + * mainnet + * preprod + * preview + * private + * sanchonet + * shelley_qa + +#### `POSTGRES_HOST` (required) + +The PostgreSQL server host to connect to. + +#### `POSTGRES_PORT` (required) + +Specifies the PostgreSQL server port to connect to. + +#### `POSTGRES_USER` (required) + +The PostgreSQL server user to connect as. + +#### `POSTGRES_PASSWORD` (required) + +Specifies the PostgreSQL server password to connect as. + +#### `SMASH_USER` (optional) + +Sets the user for HTTP basic authentication + +#### `SMASH_PASSWORD` (optional) + +Sets the password for HTTP basic authentication + +### Advanced Usage + +Excluding the `NETWORK` environment variable will simply call the `cardano-smash-server` +executable as the entrypoint, so you must pass command line arguments to provide runtime +configuration. + +```bash +docker run \ + --volume node-ipc:/node-ipc \ + --volume $PWD/environments/mainnet:/config \ + ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 \ + --config /config/db-sync-config.json --port 3100 +``` + +## Building + +Building Docker images is done with [Nix](https://nixos.org/) and requires Linux: + +```bash +nix build .#cardano-db-sync-docker +``` + +This will generate a `tar.gz` file linked to `./result` that can be loaded into docker and +run as a normal image. + +```bash +docker load < result +docker run cardano-db-sync + +## Lightweight (hybrid) image workflow + +If you prefer a smaller, standard Docker image that is easy to customise, you can use +a hybrid workflow: build a static `cardano-db-sync` binary with Nix and then build a +minimal Docker image by copying the binary into a tiny base image. + +This keeps images small (the binary is typically <100MB) and avoids shipping the NixOS +module’s network configuration inside the image. The project includes a small Docker +context in `docker/` with a minimal `Dockerfile` designed for static executables. + +Quick example (assumes a static executable): + +```bash +# Build a static executable using Nix +nix build .#cardano-db-sync-linux + +# Copy the built binary into the docker context +cp -L result/bin/cardano-db-sync docker/ + +# Build the lightweight image +docker build -t cardano-db-sync:lightweight docker/ +``` + +Notes and configuration options: + +- The `docker/Dockerfile` uses a `scratch` base image and therefore requires a fully + static binary. If you need dynamic linking, use a small base image (for example + `ubuntu:24.04` or `debian:bookworm-slim`) and replace the `FROM scratch` line. +- The image intentionally does not contain any network configuration files. To supply + configuration you have two options: + 1. Mount your configuration at runtime (recommended): + + ```bash + docker run -v $PWD/environments/mainnet:/config \ + --env POSTGRES_HOST=postgres \ + --env POSTGRES_PORT=5432 \ + cardano-db-sync:lightweight run --config /config/db-sync-config.json + ``` + + 2. Bake configuration into a derived image: create a small Dockerfile that copies + your config into the image on top of the lightweight runtime image. + +- For the canonical network configuration files (db-sync-config.json and related + assets), see the Cardano Operations Book: https://book.play.dev.cardano.org/. We + intentionally follow the same workflow used by non-containerized installations. + +Benefits: + +- Much smaller images (typically ~100MB vs 1GB+). +- Easier for non-Nix users to inspect and debug. +- Explicit configuration via volumes or derived images reduces surprises and makes + customization straightforward. + +If you plan to use the GitHub release workflow, note that the project's release +action currently builds and publishes the Nix-built Docker archive as before. The +lightweight workflow is provided as an opt-in alternative for users who want smaller +images or simpler customization. +``` + +## Running Tests with Docker PostgreSQL + +Create a `pgpass-test` file with the credentials of (taken from config/secrets/postgres_* files): + +```bash +echo "localhost:5432:cexplorer:postgres:v8hlDV0yMAHHlIurYupj" > config/pgpass-test +chmod 0600 config/pgpass-test +export PGPASSFILE=$PWD/config/pgpass-test +``` + +Start PostgreSQL via Docker Compose: + +```bash +docker compose -f docker-test.yml up +``` + +Run the migrations: + +```bash +cabal run cardano-db-tool run-migrations -- --mdir schema/ --ldir . +``` + +Run the tests: + +```bash +cabal test cardano-db +``` + +When you've finished with testing, stop and remove the test containers: + +```bash +docker compose -f docker-test.yml down +``` + From 88a607878d25c0413f7a1b536dbc9b041e8dcbe0 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Nov 2025 08:46:00 +0700 Subject: [PATCH 2/8] fix documentation --- docker/README.md | 6 +++--- docker/docker.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 08132ea2d..885ff18fc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -15,16 +15,16 @@ configuration at runtime via volumes or create their own derived images. Quickstart ---------- -1. Build a static `cardano-db-sync` binary with Nix: +1. Build the cardano-db-sync package with Nix: ```bash nix build .#cardano-db-sync-linux ``` -2. Copy the binary to this directory (follow symlinks): +2. Extract the binary from the tarball and copy it to this directory: ```bash - cp -L result/bin/cardano-db-sync docker/ + tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 bin/cardano-db-sync ``` 3. Build the image: diff --git a/docker/docker.md b/docker/docker.md index ade998529..697de7507 100644 --- a/docker/docker.md +++ b/docker/docker.md @@ -272,11 +272,11 @@ context in `docker/` with a minimal `Dockerfile` designed for static executables Quick example (assumes a static executable): ```bash -# Build a static executable using Nix +# Build the cardano-db-sync package with Nix nix build .#cardano-db-sync-linux -# Copy the built binary into the docker context -cp -L result/bin/cardano-db-sync docker/ +# Extract the binary from the tarball into the docker context +tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 bin/cardano-db-sync # Build the lightweight image docker build -t cardano-db-sync:lightweight docker/ From ac66b37b1273e4a4411dc4d29a8bf14632630fe2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Nov 2025 08:54:18 +0700 Subject: [PATCH 3/8] fix docs --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 885ff18fc..f69e23d2d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -24,7 +24,7 @@ Quickstart 2. Extract the binary from the tarball and copy it to this directory: ```bash - tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 bin/cardano-db-sync + tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 ./bin/cardano-db-sync ``` 3. Build the image: From 33bf95a96486441d9d5cb9154436e351f3fdbb84 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Nov 2025 09:15:08 +0700 Subject: [PATCH 4/8] fix docs --- docker/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index f69e23d2d..9eebcba49 100644 --- a/docker/README.md +++ b/docker/README.md @@ -21,12 +21,18 @@ Quickstart nix build .#cardano-db-sync-linux ``` -2. Extract the binary from the tarball and copy it to this directory: +2. a) Extract the binary from the tarball and copy it to this directory: ```bash tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 ./bin/cardano-db-sync ``` +2. b) move the binary: + + ```bash + mv docker/bin/cardano-db-sync ./docker && rm -r docker/bin/ + ``` + 3. Build the image: ```bash From 92847c85ef70b8661c869e1b986f030decd02183 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Nov 2025 09:18:15 +0700 Subject: [PATCH 5/8] fix docs --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 9eebcba49..29fe2f4ab 100644 --- a/docker/README.md +++ b/docker/README.md @@ -44,7 +44,7 @@ Quickstart ```bash docker run -v $PWD/environments/mainnet:/config \ --env POSTGRES_HOST=postgres --env POSTGRES_PORT=5432 \ - cardano-db-sync:lightweight run --config /config/db-sync-config.json + cardano-db-sync:lightweight --config /config/db-sync-config.json ``` Notes From ce733005ba52e92c0c620a46077538e46eb3e81e Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 29 Nov 2025 08:18:39 +0700 Subject: [PATCH 6/8] add build script, fix dockerfile, draft docs --- docker/Dockerfile | 32 +++++++++++++++++--------------- docker/README.md | 9 +++++++-- docker/build.sh | 12 ++++++++++++ 3 files changed, 36 insertions(+), 17 deletions(-) create mode 100755 docker/build.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 7e1f2cace..1be97396f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,25 +3,27 @@ # Usage (example): # 1. Build a static executable with Nix: # nix build .#cardano-db-sync-linux -# 2. Copy the executable into this folder (follow symlinks): -# cp -L result/bin/cardano-db-sync docker/ -# 3. Build the image: -# docker build -t cardano-db-sync:lightweight docker/ +# 2. Build the image from the project root: +# docker build -f docker/Dockerfile -t cardano-db-sync:lightweight . +# +# Note: Since result/ is a Nix symlink, Docker may have issues following it. +# If this fails, you can dereference the symlink first: +# mkdir -p docker/build && cp -L result/*.tar.gz docker/build/ +# docker build -f docker/Dockerfile -t cardano-db-sync:lightweight . # # Important: this Dockerfile assumes the binary is a fully static executable # (statically linked). If it's not static, use a small base image (see docs). -FROM scratch - -# Add the static executable produced by Nix. The builder step places -# the binary next to this Dockerfile before running `docker build`. -COPY cardano-db-sync /usr/local/bin/cardano-db-sync +# Stage 1: Extract the binary from the tarball +FROM busybox:latest AS extractor +ARG TARBALL +# ADD automatically extracts tarballs +ADD ${TARBALL} /tmp/extract/ +RUN find /tmp/extract -name cardano-db-sync -type f -exec mv {} /cardano-db-sync \; && \ + chmod +x /cardano-db-sync -# Minimal entrypoint; configuration should be supplied via mounted files -# or environment variables. We avoid baked-in configs to keep images small -# and easier to customize. +# Stage 2: Minimal runtime image +FROM scratch +COPY --from=extractor /cardano-db-sync /usr/local/bin/cardano-db-sync ENTRYPOINT ["/usr/local/bin/cardano-db-sync"] - -# Default to printing help if no args supplied (preserves normal CLI behaviour) CMD ["--help"] - diff --git a/docker/README.md b/docker/README.md index 29fe2f4ab..ec832bc45 100644 --- a/docker/README.md +++ b/docker/README.md @@ -39,12 +39,17 @@ Quickstart docker build -t cardano-db-sync:lightweight docker/ ``` +4. get the configs: +- https://book.world.dev.cardano.org/environments/preprod/db-sync-config.json + 4. Run the container and mount your configuration (recommended): ```bash - docker run -v $PWD/environments/mainnet:/config \ + docker run -v $PWD/environments/preprod:/config \ --env POSTGRES_HOST=postgres --env POSTGRES_PORT=5432 \ - cardano-db-sync:lightweight --config /config/db-sync-config.json + cardano-db-sync:lightweight --config $PWD/config/db-sync-config.json \ + --socket-path "$HOME/workshop/cardano-node/configuration/preprod/db/node.socket" \ + --schema-dir /schema ``` Notes diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 000000000..a720b4886 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dereference the Nix result symlink +TARBALL=$(basename "$(readlink -f result/*.tar.gz)") +cp -L result/*.tar.gz docker/ + +# Build with the exact tarball name as build arg +docker build --build-arg TARBALL="$TARBALL" -t cardano-db-sync:lightweight docker/ + +# Cleanup +rm -f docker/*.tar.gz From 9e7564625678970a0f24df1109a1473f049b33b8 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 2 Jan 2026 11:41:13 +0700 Subject: [PATCH 7/8] change base image, improve documentation --- docker/Dockerfile | 2 +- docker/README.md | 60 +------- docker/docker.md | 354 ---------------------------------------------- 3 files changed, 3 insertions(+), 413 deletions(-) delete mode 100644 docker/docker.md diff --git a/docker/Dockerfile b/docker/Dockerfile index 1be97396f..08fb2f8fa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,7 +23,7 @@ RUN find /tmp/extract -name cardano-db-sync -type f -exec mv {} /cardano-db-sync chmod +x /cardano-db-sync # Stage 2: Minimal runtime image -FROM scratch +FROM ubuntu:24.04 COPY --from=extractor /cardano-db-sync /usr/local/bin/cardano-db-sync ENTRYPOINT ["/usr/local/bin/cardano-db-sync"] CMD ["--help"] diff --git a/docker/README.md b/docker/README.md index ec832bc45..d06b34099 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,62 +4,6 @@ Lightweight Docker image for cardano-db-sync This folder contains a minimal Docker context and `Dockerfile` intended for use with a statically-built `cardano-db-sync` executable produced by Nix. -Why this exists ---------------- - -Building the full NixOS-based Docker image is a reproducible approach, but can lead -to large images and bundled configuration that make customization more difficult. -This lightweight approach lets users build a small, standard Docker image and supply -configuration at runtime via volumes or create their own derived images. - -Quickstart ----------- - -1. Build the cardano-db-sync package with Nix: - - ```bash - nix build .#cardano-db-sync-linux - ``` - -2. a) Extract the binary from the tarball and copy it to this directory: - - ```bash - tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 ./bin/cardano-db-sync - ``` - -2. b) move the binary: - - ```bash - mv docker/bin/cardano-db-sync ./docker && rm -r docker/bin/ - ``` - -3. Build the image: - - ```bash - docker build -t cardano-db-sync:lightweight docker/ - ``` - -4. get the configs: -- https://book.world.dev.cardano.org/environments/preprod/db-sync-config.json - -4. Run the container and mount your configuration (recommended): - - ```bash - docker run -v $PWD/environments/preprod:/config \ - --env POSTGRES_HOST=postgres --env POSTGRES_PORT=5432 \ - cardano-db-sync:lightweight --config $PWD/config/db-sync-config.json \ - --socket-path "$HOME/workshop/cardano-node/configuration/preprod/db/node.socket" \ - --schema-dir /schema - ``` - -Notes ------ - -- The `Dockerfile` uses `FROM scratch` and therefore requires the binary to be - statically linked. If you need dynamic linking, change the base image to a small - distro such as `ubuntu:24.04` or `debian:bookworm-slim`. -- The image intentionally omits network configuration files. Use volumes or a - derived image if you want to embed configs. - -See also: the project's `doc/docker.md` for more details and guidance. +## testing +docker run --rm cardano-db-sync:lightweight version diff --git a/docker/docker.md b/docker/docker.md deleted file mode 100644 index 697de7507..000000000 --- a/docker/docker.md +++ /dev/null @@ -1,354 +0,0 @@ -# Docker - -## Quickstart - -We provide an example -[docker-compose.yml](https://github.com/IntersectMBO/cardano-db-sync/blob/master/docker-compose.example.yml) -to quickly get set up with `cardano-db-sync`. Keep in mind that this is only a template, -and users are encouraged to tweak it to meet their needs. - -It is not recommended to use this for a production set up, and should only be -used for local development. - -Create a working directory: - -```bash -mkdir ~/src -cd ~/src -``` - -Download the example Docker Compose file: - -```bash -curl -o docker-compose.yml \ - https://raw.githubusercontent.com/IntersectMBO/cardano-db-sync/master/docker-compose.example.yml -``` - -Start `cardano-node`, `postgresql`, and `cardano-db-sync` services: - -```bash -docker compose up -d && docker compose logs -f -``` - -The PostgreSQL database should now be exposed on localhost port `5432` - -To connect to PostgreSQL database: - -```bash -psql -h 0.0.0.0 -p 5432 -U postgres -d cexplorer -``` - -### Connecting to a Other Networks - -To connect to different network use `NETWORK` environment variable: - -```bash -NETWORK=preprod docker compose up -d && docker compose logs -f -``` - -## Running `cardano-db-sync` - -Start `cardano-db-sync`: - -```bash -docker run \ - --env NETWORK=mainnet \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - --volume db-sync-data:/var/lib/cexplorer \ - --volume node-ipc \ - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 -``` - -### Environment Variables - -#### `NETWORK` - -Specifies a network to connect to. If specified, `cardano-db-sync` will provide the -configuration files. If not specified, will call the `cardano-db-sync` executable as the -entrypoint. Possible values are: - - * mainnet - * preprod - * preview - * private - * sanchonet - * shelley_qa - -#### `POSTGRES_HOST` (required) - -The PostgreSQL server host to connect to. - -#### `POSTGRES_PORT` (required) - -Specifies the PostgreSQL server port to connect to. - -#### `POSTGRES_USER` (required) - -The PostgreSQL server user to connect as. - -#### `POSTGRES_PASSWORD` (required) - -Specifies the PostgreSQL server password to connect as. - -#### `RESTORE_SNAPSHOT` (optional) - -Specifies a `cardano-db-sync` snapshot archive to download and restore from. If omitted, -it will sync from genesis. see [Restoring From Snapshots](#restoring-from-snapshots) - -### `DB_SYNC_CONFIG` (optional) - -Overrides the `db-sync-config.json` provided by the network configuration. See [Overriding -Network Configuration](#overriding-network-configuration). - -#### `EXTRA_DB_SYNC_ARGS` (optional) - -Extra command line arguments to pass to `cardano-db-sync`. For example: - -```bash -docker run \ - --env NETWORK=mainnet \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - --env EXTRA_DB_SYNC_ARGS="--skip-fix" - --volume db-sync-data:/var/lib/cexplorer \ - --volume node-ipc \ - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 -``` - -### Overriding Network Configuration - -Overriding the configuration file can be done by passing the `DB_SYNC_CONFIG` environment -variable: - -```bash -docker run \ - --env NETWORK=mainnet \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - --env DB_SYNC_CONFIG=/config/db-sync-config.json \ - --volume db-sync-data:/var/lib/cexplorer \ - --volume node-ipc:/node-ipc \ - --volume $PWD/environments/mainnet:/config \ - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 -``` - -### Restoring From Snapshots - -Snapshots are provided for mainnet to restore the PostgreSQL database. See the [latest -releases](https://github.com/IntersectMBO/cardano-db-sync/releases) for a recent snapshot -matched with the `cardano-db-sync` version. - -Although, you can specify a URL directly, we recommend downloading the image separately to avoid -having to download it multiple times in case of failure. First, download the latest snapshot: - -```bash -curl -O https://update-cardano-mainnet.iohk.io/cardano-db-sync/13.3/db-sync-snapshot-schema-13.3-block-10611621-x86_64.tgz -``` - -Restore the snapshot with `RESTORE_SNAPSHOT`: - -```bash -docker run \ - --env NETWORK=mainnet \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - --env RESTORE_SNAPSHOT=/data/db-sync-snapshot-schema-13.3-block-10611621-x86_64.tgz \ - --volume db-sync-data:/var/lib/cexplorer \ - --volume node-ipc:/node-ipc \ - --volume $PWD:/data - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 -``` - -### Advanced Usage - -Excluding the `NETWORK` environment variable will simply call the `cardano-db-sync` -executable as the entrypoint, so you must pass command line arguments to projvide runtime -configuration. The `--schema-dir` argument is automatically set and is not required. - -```bash -docker run \ - --volume db-sync-data:/var/lib/cexplorer \ - --volume node-ipc:/node-ipc \ - --volume $PWD/environments/mainnet:/config \ - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 \ - run --config /config/db-sync-config.json --socket-path /node-ipc/node.socket -``` - -## Running `cardano-smash-server` - -Start `cardano-smash-server`: - -```bash -docker run \ - --env NETWORK=mainnet \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - --env SMASH_USER=smash-admin \ - --env SMASH_PASSWORD=smash-password \ - --publish 3100:3100 \ - --volume node-ipc:/node-ipc \ - ghcr.io/IntersectMBO/cardano-smash-server:13.3.0.0 -``` - -### Environment Variables - -#### `NETWORK` (optional) - -Specifies a network to connect to. If specified, `cardano-db-sync` will provide the -configuration files. If not specified, will call the `cardano-db-sync` executable as the -entrypoint. Possible values are: - - * mainnet - * preprod - * preview - * private - * sanchonet - * shelley_qa - -#### `POSTGRES_HOST` (required) - -The PostgreSQL server host to connect to. - -#### `POSTGRES_PORT` (required) - -Specifies the PostgreSQL server port to connect to. - -#### `POSTGRES_USER` (required) - -The PostgreSQL server user to connect as. - -#### `POSTGRES_PASSWORD` (required) - -Specifies the PostgreSQL server password to connect as. - -#### `SMASH_USER` (optional) - -Sets the user for HTTP basic authentication - -#### `SMASH_PASSWORD` (optional) - -Sets the password for HTTP basic authentication - -### Advanced Usage - -Excluding the `NETWORK` environment variable will simply call the `cardano-smash-server` -executable as the entrypoint, so you must pass command line arguments to provide runtime -configuration. - -```bash -docker run \ - --volume node-ipc:/node-ipc \ - --volume $PWD/environments/mainnet:/config \ - ghcr.io/IntersectMBO/cardano-db-sync:13.3.0.0 \ - --config /config/db-sync-config.json --port 3100 -``` - -## Building - -Building Docker images is done with [Nix](https://nixos.org/) and requires Linux: - -```bash -nix build .#cardano-db-sync-docker -``` - -This will generate a `tar.gz` file linked to `./result` that can be loaded into docker and -run as a normal image. - -```bash -docker load < result -docker run cardano-db-sync - -## Lightweight (hybrid) image workflow - -If you prefer a smaller, standard Docker image that is easy to customise, you can use -a hybrid workflow: build a static `cardano-db-sync` binary with Nix and then build a -minimal Docker image by copying the binary into a tiny base image. - -This keeps images small (the binary is typically <100MB) and avoids shipping the NixOS -module’s network configuration inside the image. The project includes a small Docker -context in `docker/` with a minimal `Dockerfile` designed for static executables. - -Quick example (assumes a static executable): - -```bash -# Build the cardano-db-sync package with Nix -nix build .#cardano-db-sync-linux - -# Extract the binary from the tarball into the docker context -tar xf result/cardano-db-sync-*-linux.tar.gz -C docker/ --strip-components=1 bin/cardano-db-sync - -# Build the lightweight image -docker build -t cardano-db-sync:lightweight docker/ -``` - -Notes and configuration options: - -- The `docker/Dockerfile` uses a `scratch` base image and therefore requires a fully - static binary. If you need dynamic linking, use a small base image (for example - `ubuntu:24.04` or `debian:bookworm-slim`) and replace the `FROM scratch` line. -- The image intentionally does not contain any network configuration files. To supply - configuration you have two options: - 1. Mount your configuration at runtime (recommended): - - ```bash - docker run -v $PWD/environments/mainnet:/config \ - --env POSTGRES_HOST=postgres \ - --env POSTGRES_PORT=5432 \ - cardano-db-sync:lightweight run --config /config/db-sync-config.json - ``` - - 2. Bake configuration into a derived image: create a small Dockerfile that copies - your config into the image on top of the lightweight runtime image. - -- For the canonical network configuration files (db-sync-config.json and related - assets), see the Cardano Operations Book: https://book.play.dev.cardano.org/. We - intentionally follow the same workflow used by non-containerized installations. - -Benefits: - -- Much smaller images (typically ~100MB vs 1GB+). -- Easier for non-Nix users to inspect and debug. -- Explicit configuration via volumes or derived images reduces surprises and makes - customization straightforward. - -If you plan to use the GitHub release workflow, note that the project's release -action currently builds and publishes the Nix-built Docker archive as before. The -lightweight workflow is provided as an opt-in alternative for users who want smaller -images or simpler customization. -``` - -## Running Tests with Docker PostgreSQL - -Create a `pgpass-test` file with the credentials of (taken from config/secrets/postgres_* files): - -```bash -echo "localhost:5432:cexplorer:postgres:v8hlDV0yMAHHlIurYupj" > config/pgpass-test -chmod 0600 config/pgpass-test -export PGPASSFILE=$PWD/config/pgpass-test -``` - -Start PostgreSQL via Docker Compose: - -```bash -docker compose -f docker-test.yml up -``` - -Run the migrations: - -```bash -cabal run cardano-db-tool run-migrations -- --mdir schema/ --ldir . -``` - -Run the tests: - -```bash -cabal test cardano-db -``` - -When you've finished with testing, stop and remove the test containers: - -```bash -docker compose -f docker-test.yml down -``` - From 73cbfba103a0b5af5d8929d03f2c70a8542feee1 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 2 Jan 2026 12:02:09 +0700 Subject: [PATCH 8/8] feat(docker): add lightweight cardano-db-sync image with baked schema verification - Copy cardano-db-sync binary into a minimal Ubuntu image. - Bake schema directory into the image and embed a deterministic SCHEMA_HASH. - Add entrypoint template that verifies the runtime schema checksum matches the baked hash and execs the binary with --schema-dir /schema. - Replace fragile heredoc with a template + sed substitution in the Dockerfile. - Add build script to copy tarball/schema into build context and compute SCHEMA_HASH. - Add test-image.sh to validate image behavior (help, schema mount, modified-schema detection, binary presence). --- docker/Dockerfile | 19 +++- docker/build.sh | 15 ++- docker/docker-entrypoint.sh.template | 13 +++ docker/test-image.sh | 131 +++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 docker/docker-entrypoint.sh.template create mode 100755 docker/test-image.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 08fb2f8fa..cbad41a25 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,6 +24,23 @@ RUN find /tmp/extract -name cardano-db-sync -type f -exec mv {} /cardano-db-sync # Stage 2: Minimal runtime image FROM ubuntu:24.04 +ARG SCHEMA_HASH + +# copy binary from extractor COPY --from=extractor /cardano-db-sync /usr/local/bin/cardano-db-sync -ENTRYPOINT ["/usr/local/bin/cardano-db-sync"] + +# copy schema directory (must be present in build context) +COPY schema /schema +RUN chmod -R 755 /schema + +# copy entrypoint template then substitute the baked SCHEMA_HASH into it +COPY docker-entrypoint.sh.template /usr/local/bin/docker-entrypoint.sh.template + +# Use sed to bake the SCHEMA_HASH into the entrypoint script, then cleanup template +RUN sed "s|@SCHEMA_HASH@|${SCHEMA_HASH}|g" /usr/local/bin/docker-entrypoint.sh.template \ + > /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-entrypoint.sh && \ + rm -f /usr/local/bin/docker-entrypoint.sh.template + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] CMD ["--help"] diff --git a/docker/build.sh b/docker/build.sh index a720b4886..69c7dd6ab 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,12 +1,21 @@ #!/usr/bin/env bash set -euo pipefail -# Dereference the Nix result symlink +# Dereference the Nix result symlink and get the tarball basename TARBALL=$(basename "$(readlink -f result/*.tar.gz)") + +# copy tarball into docker build context cp -L result/*.tar.gz docker/ -# Build with the exact tarball name as build arg -docker build --build-arg TARBALL="$TARBALL" -t cardano-db-sync:lightweight docker/ +# copy runtime schema into build context so it gets baked into the image +cp -a schema docker/ + +# compute a deterministic schema checksum (sorted file list -> combined sha) +SCHEMA_HASH=$(cd schema && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}') + +# Build with the exact tarball name and schema hash as build args +docker build --build-arg TARBALL="$TARBALL" --build-arg SCHEMA_HASH="$SCHEMA_HASH" -t cardano-db-sync:lightweight docker/ # Cleanup rm -f docker/*.tar.gz +rm -rf docker/schema diff --git a/docker/docker-entrypoint.sh.template b/docker/docker-entrypoint.sh.template new file mode 100644 index 000000000..c9ca2bda9 --- /dev/null +++ b/docker/docker-entrypoint.sh.template @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eu + +# compute runtime checksum deterministically (sorted file list) +runtime_hash=$(cd /schema && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}') + +# baked checksum will be substituted at build time +if [ "$runtime_hash" != "@SCHEMA_HASH@" ]; then + echo "schema checksum mismatch: runtime=$runtime_hash baked=@SCHEMA_HASH@" >&2 + exit 1 +fi + +exec /usr/local/bin/cardano-db-sync --schema-dir /schema "$@" \ No newline at end of file diff --git a/docker/test-image.sh b/docker/test-image.sh new file mode 100755 index 000000000..cdf0eaa70 --- /dev/null +++ b/docker/test-image.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Test the cardano-db-sync Docker image: +# - basic invocation (defaults to --help) +# - running with the original schema directory (should succeed) +# - running with a modified schema (should fail and report checksum mismatch) +# - check that the binary exists by bypassing the ENTRYPOINT +# +# Usage: ./test-image.sh [IMAGE] [SCHEMA_DIR] +# Defaults: +# IMAGE=cardano-db-sync:lightweight +# SCHEMA_DIR=./schema + +set -uo pipefail + +IMAGE="${1:-cardano-db-sync:lightweight}" +SCHEMA_DIR="${2:-./schema}" + +TMPDIR="" +FAILED=0 + +cleanup() { + rc=$? + if [[ -n "$TMPDIR" && -d "$TMPDIR" ]]; then + rm -rf "$TMPDIR" + fi + exit $rc +} +trap cleanup EXIT + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { echo "required command '$1' not found" >&2; exit 2; } +} + +require_cmd docker +require_cmd mktemp +require_cmd cp +require_cmd sed +require_cmd realpath + +echo "Image: $IMAGE" +echo "Schema dir: $SCHEMA_DIR" + +if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + echo "ERROR: Docker image '$IMAGE' not found locally." >&2 + echo "Either build it or pull it before running this test script." >&2 + exit 3 +fi + +run_container_capture() { + # args: container args... + docker run --rm "$@" 2>&1 + return ${PIPESTATUS[0]:-0} +} + +echo +echo "1) Basic invocation (no extra mounts) - should exit 0 (show help)" +out=$(run_container_capture "$IMAGE") +rc=$? +if [[ $rc -eq 0 ]]; then + echo "PASS: container exited 0" +else + echo "FAIL: container exited with $rc" + echo "Output:" + echo "-----" + echo "$out" + echo "-----" + FAILED=$((FAILED+1)) +fi + +if [[ ! -d "$SCHEMA_DIR" ]]; then + echo + echo "WARNING: schema directory '$SCHEMA_DIR' not found; skipping schema-related tests." +else + echo + echo "2) Run with host schema mounted to /schema (expected: success)" + out=$(run_container_capture -v "$(realpath "$SCHEMA_DIR")":/schema:ro "$IMAGE") + rc=$? + if [[ $rc -eq 0 ]]; then + echo "PASS: container exited 0 with host schema" + else + echo "FAIL: container exited $rc with host schema" + echo "Output:" + echo "-----" + echo "$out" + echo "-----" + FAILED=$((FAILED+1)) + fi + + echo + echo "3) Run with a modified schema (expected: schema checksum mismatch -> non-zero and error text)" + TMPDIR=$(mktemp -d) + cp -a "$SCHEMA_DIR"/. "$TMPDIR"/ + # Modify the schema to change checksum (add a small file) + echo "injected-change-$(date +%s)" > "$TMPDIR"/.injected-change + + out=$(run_container_capture -v "$TMPDIR":/schema:ro "$IMAGE") + rc=$? + if [[ $rc -ne 0 && "$out" == *"schema checksum mismatch"* ]]; then + echo "PASS: schema mismatch detected as expected (exit $rc)" + else + echo "FAIL: expected schema mismatch (non-zero exit and 'schema checksum mismatch' message)." + echo "Exit code: $rc" + echo "Output:" + echo "-----" + echo "$out" + echo "-----" + FAILED=$((FAILED+1)) + fi +fi + +echo +echo "4) Binary presence check inside image (override ENTRYPOINT so we can inspect fs)" +# IMPORTANT: override the image ENTRYPOINT so our test runs directly in the container +if docker run --rm --entrypoint /bin/sh "$IMAGE" -c 'test -x /usr/local/bin/cardano-db-sync' >/dev/null 2>&1; then + echo "PASS: /usr/local/bin/cardano-db-sync exists and is executable (entrypoint bypassed)" + echo " File info:" + docker run --rm --entrypoint /bin/sh "$IMAGE" -c 'ls -la /usr/local/bin/cardano-db-sync || true; file /usr/local/bin/cardano-db-sync || true' +else + echo "FAIL: /usr/local/bin/cardano-db-sync missing or not executable (even with entrypoint bypassed)" + FAILED=$((FAILED+1)) +fi + +echo +if [[ $FAILED -eq 0 ]]; then + echo "ALL TESTS PASSED" +else + echo "SOME TESTS FAILED: $FAILED failure(s)" + exit 4 +fi + +# cleanup happens via trap