From 00d34192333752a02928e04b22b18c5f22f79b10 Mon Sep 17 00:00:00 2001
From: reo101 <pavel.atanasov2001@gmail.com>
Date: Mon, 30 Jun 2025 11:44:16 +0300
Subject: [PATCH 1/4] feat(modules): Initial `healthcheck` module

Co-authored-by: Simeon Armenchev <monyarm@gmail.com>
Co-authored-by: Martin Nikov <mnikov12@gmail.com>
---
 modules/default.nix             |   1 +
 modules/healthcheck/default.nix | 248 ++++++++++++++++++++++++++++++++
 2 files changed, 249 insertions(+)
 create mode 100644 modules/healthcheck/default.nix

diff --git a/modules/default.nix b/modules/default.nix
index 5698129d..2e4cd37f 100644
--- a/modules/default.nix
+++ b/modules/default.nix
@@ -9,5 +9,6 @@
     ./secrets.nix
     ./mcl-disko
     ./pharos
+    ./healthcheck
   ];
 }
diff --git a/modules/healthcheck/default.nix b/modules/healthcheck/default.nix
new file mode 100644
index 00000000..f9a34d0f
--- /dev/null
+++ b/modules/healthcheck/default.nix
@@ -0,0 +1,248 @@
+{ ... }:
+{
+  flake.modules.nixos.healthcheck =
+    {
+      config,
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      inherit (lib) types;
+      mkProbeOptions = x: {
+        options =
+          {
+            enable = lib.mkEnableOption "the ${x} probe";
+
+            command = lib.mkOption {
+              type = types.str;
+              description = "The command to execute for the ${x} check. Any necessary programs should be added to the healthcheck.runtimePackages option.";
+            };
+
+            initialDelay = lib.mkOption {
+              type = types.int;
+              default = 15;
+              description = "Seconds to wait after the service is up before the first ${x} probe.";
+            };
+
+            interval = lib.mkOption {
+              type = types.int;
+              default = if x == "liveness" then 30 else 2;
+              description = "How often (in seconds) to perform the ${x} probe.";
+            };
+
+            timeout = lib.mkOption {
+              type = types.int;
+              default = 10;
+              description = "Seconds after which the ${x} probe command times out.";
+            };
+
+            # TODO: `{success,failure}_treshold`
+
+            retryCount = lib.mkOption {
+              type = types.int;
+              default = 10;
+              description = "Number of times to retry the ${x} probe before considering it failed. (-1 means infinite retries)";
+            };
+          }
+          // lib.optionalAttrs (x == "readiness") {
+            statusWaitingMessage = lib.mkOption {
+              type = types.str;
+              default = "Service starting, waiting for ready signal...";
+              description = "The status message to send to systemd while waiting.";
+            };
+
+            statusReadyMessage = lib.mkOption {
+              type = types.str;
+              default = "Service is ready.";
+              description = ''
+                The status message to send when the service is ready.
+                Use %OUTPUT% to substitute the output of the check command.
+              '';
+            };
+          };
+      };
+
+      # Options for the liveness probe (timer-based check)
+      livenessProbeOptions = mkProbeOptions "liveness";
+
+      # Options for the readiness probe (notify-based check)
+      readinessProbeOptions = mkProbeOptions "readiness";
+    in
+    {
+      config =
+        let
+          servicesWithHealthcheck = lib.filterAttrs (
+            _name: service: service.healthcheck != null
+          ) config.mcl.services;
+        in
+        {
+          assertions = lib.pipe config.mcl.services [
+            (lib.filterAttrs (_: service: service.healthcheck != null))
+            (lib.mapAttrsToList (
+              name: _:
+              let
+                serviceConfig = config.systemd.services.${name}.serviceConfig;
+              in
+              {
+                # NOTE: as per <https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStartPost=>
+                assertion = lib.elem serviceConfig.Type [
+                  "simple"
+                  "idle"
+                ];
+                message = ''
+                  Service ${name} is not of type "simple" or "idle", but ${serviceConfig.Type}.
+                  Cannot attach a readiness probe to it.
+                '';
+              }
+            ))
+          ];
+          systemd = {
+            services =
+              let
+                mainServices = lib.mapAttrs (
+                  mainServiceName: serviceConfig:
+                  let
+                    cfg = serviceConfig.healthcheck;
+                    probeCfg = cfg.readiness-probe;
+                  in
+                  lib.mkIf (cfg != null && probeCfg.enable) {
+                    # Timeout is now handled manually by the new `ExecStartPost`
+                    serviceConfig.TimeoutStartSec = "infinity";
+
+                    # Add an `ExecStartPost` with a script that runs the readiness probe
+                    # WARN: cannot assure that there is no `ExecStartPost` in the original `serviceConfig`
+                    #       (in order to avoid overriding/duplication)
+                    serviceConfig.ExecStartPost =
+                      let
+                        scriptPath = lib.makeBinPath (cfg.runtimePackages ++ (serviceConfig.path or [ ]));
+                      in
+                      lib.getExe (
+                        pkgs.writeShellScriptBin "${mainServiceName}-readiness-check" ''
+                          set -o nounset
+
+                          export PATH="${scriptPath}:$PATH"
+
+                          echo "Health check: starting background readiness probe for ${mainServiceName}."
+                          sleep ${toString probeCfg.initialDelay}
+                          retryCount=${toString probeCfg.retryCount}
+                          while true; do
+                            if (timeout ${toString probeCfg.timeout}s ${probeCfg.command} &> /dev/null); then
+                              echo "Health check: probe successful. Notifying systemd that the service is ready."
+                              exit 0
+                            else
+                              echo "Health check: probe not successful. Notifying systemd that the service is still waiting. Retrying in ${toString probeCfg.interval} seconds..."
+                              if [[ ''${retryCount} -ne -1 ]]; then
+                                retryCount=$((retryCount - 1))
+                                if [[ ''${retryCount} -le 0 ]]; then
+                                  echo "Health check: probe failed after maximum retries. Exiting."
+                                  exit 1
+                                fi
+                              fi
+                            fi
+                            sleep ${toString probeCfg.interval}
+                          done
+                        ''
+                      );
+                  }
+                ) servicesWithHealthcheck;
+
+                healthCheckServices = lib.mapAttrs' (
+                  mainServiceName: serviceConfig:
+                  let
+                    cfg = serviceConfig.healthcheck;
+                  in
+                  {
+                    name = "${mainServiceName}-liveness-check";
+                    value = lib.mkIf (cfg != null && cfg.liveness-probe.enable) (
+                      let
+                        probeCfg = cfg.liveness-probe;
+                        checkScript = pkgs.writeShellScriptBin "liveness-check" ''
+                          #!${pkgs.runtimeShell}
+                          echo "Executing liveness probe for ${mainServiceName}..."
+                          if ! (timeout ${toString probeCfg.timeout}s ${probeCfg.command} &> /dev/null); then
+                            echo "Liveness probe for ${mainServiceName} failed. Triggering restart..."
+                            ${lib.getExe' pkgs.systemd "systemctl"} restart ${lib.escapeShellArg mainServiceName}.service
+                            exit 1
+                          fi
+                          echo "Liveness probe for ${mainServiceName} successful."
+                        '';
+                      in
+                      {
+                        description = "Liveness check for ${mainServiceName}";
+                        path = cfg.runtimePackages;
+                        serviceConfig = {
+                          Type = "oneshot";
+                          ExecStart = "${lib.getExe checkScript}";
+                        };
+                      }
+                    );
+                  }
+                ) servicesWithHealthcheck;
+              in
+              mainServices // healthCheckServices;
+
+            timers = lib.mapAttrs' (
+              mainServiceName: serviceConfig:
+              let
+                cfg = serviceConfig.healthcheck;
+              in
+              {
+                name = "${mainServiceName}-liveness-check";
+                value = lib.mkIf (cfg != null && cfg.liveness-probe.enable) (
+                  let
+                    probeCfg = cfg.liveness-probe;
+                  in
+                  {
+                    description = "Timer for ${mainServiceName} liveness probe";
+                    wantedBy = [ "timers.target" ];
+                    timerConfig = {
+                      OnActiveSec = "${toString probeCfg.initialDelay}s";
+                      OnUnitInactiveSec = "${toString probeCfg.interval}s";
+                    };
+                  }
+                );
+              }
+            ) servicesWithHealthcheck;
+          };
+        };
+
+      options.mcl.services = lib.mkOption {
+        default = { };
+        type = types.attrsOf (
+          types.submodule {
+            options = {
+              healthcheck = lib.mkOption {
+                default = null;
+                description = "Declarative health checks for this systemd service.";
+                type = types.nullOr (
+                  types.submodule {
+                    options = {
+                      # Programs to add to the PATH for the health check.
+                      runtimePackages = lib.mkOption {
+                        type = types.listOf types.package;
+                        default = [ ];
+                        description = "Additional programs to add to the PATH for health checks.";
+                      };
+
+                      # The new readiness probe that uses the notify pattern.
+                      readiness-probe = lib.mkOption {
+                        type = types.submodule readinessProbeOptions;
+                        default = { };
+                      };
+
+                      # The liveness probe (timer-based).
+                      liveness-probe = lib.mkOption {
+                        type = types.submodule livenessProbeOptions;
+                        default = { };
+                      };
+                    };
+                  }
+                );
+              };
+            };
+          }
+        );
+      };
+    };
+}

From af2e77f9423e1d9da722967f7323b2a4b4dc022d Mon Sep 17 00:00:00 2001
From: Simeon Armenchev <monyarm@gmail.com>
Date: Mon, 7 Jul 2025 13:55:57 +0300
Subject: [PATCH 2/4] feat(machines): add machine per nixos module

---
 flake.nix                                     |  1 +
 machines/blankpass.txt                        |  0
 machines/default.nix                          | 22 ++++++++
 machines/modules/all.nix                      | 18 +++++++
 machines/modules/base.nix                     | 15 ++++++
 .../ethereum-validators-monitoring.nix        | 20 +++++++
 machines/modules/folder-size-metrics.nix      | 15 ++++++
 machines/modules/healthcheck.nix              | 53 +++++++++++++++++++
 machines/modules/lido-keys-api.nix            | 30 +++++++++++
 machines/modules/lido-validator-ejector.nix   | 25 +++++++++
 .../modules/lido-withdrawals-automation.nix   | 16 ++++++
 machines/modules/mcl-disko.nix                |  8 +++
 machines/modules/mcl-host-info.nix            | 12 +++++
 machines/modules/mcl-secrets.nix              |  8 +++
 machines/modules/pharos.nix                   | 11 ++++
 machines/modules/pyroscope.nix                | 10 ++++
 machines/modules/random-alerts.nix            |  8 +++
 17 files changed, 272 insertions(+)
 create mode 100644 machines/blankpass.txt
 create mode 100644 machines/default.nix
 create mode 100644 machines/modules/all.nix
 create mode 100644 machines/modules/base.nix
 create mode 100644 machines/modules/ethereum-validators-monitoring.nix
 create mode 100644 machines/modules/folder-size-metrics.nix
 create mode 100644 machines/modules/healthcheck.nix
 create mode 100644 machines/modules/lido-keys-api.nix
 create mode 100644 machines/modules/lido-validator-ejector.nix
 create mode 100644 machines/modules/lido-withdrawals-automation.nix
 create mode 100644 machines/modules/mcl-disko.nix
 create mode 100644 machines/modules/mcl-host-info.nix
 create mode 100644 machines/modules/mcl-secrets.nix
 create mode 100644 machines/modules/pharos.nix
 create mode 100644 machines/modules/pyroscope.nix
 create mode 100644 machines/modules/random-alerts.nix

diff --git a/flake.nix b/flake.nix
index d38132a3..d84e1889 100644
--- a/flake.nix
+++ b/flake.nix
@@ -221,6 +221,7 @@
         ./checks
         ./modules
         ./packages
+        ./machines
         ./shells
       ];
       systems = [
diff --git a/machines/blankpass.txt b/machines/blankpass.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/machines/default.nix b/machines/default.nix
new file mode 100644
index 00000000..a83b570d
--- /dev/null
+++ b/machines/default.nix
@@ -0,0 +1,22 @@
+{
+  lib,
+  self,
+  ...
+}:
+let
+  system = "x86_64-linux";
+  mkMachine = x: {
+    "${lib.removeSuffix ".nix" x}" = lib.nixosSystem {
+      specialArgs = { inherit self system; };
+      modules = lib.unique [
+        ./modules/base.nix
+        ./modules/${x}
+      ];
+    };
+  };
+in
+{
+  flake.nixosConfigurations = (
+    lib.mergeAttrsList (lib.map (x: mkMachine x) (builtins.attrNames (builtins.readDir ./modules)))
+  );
+}
diff --git a/machines/modules/all.nix b/machines/modules/all.nix
new file mode 100644
index 00000000..e9800176
--- /dev/null
+++ b/machines/modules/all.nix
@@ -0,0 +1,18 @@
+{
+  self,
+  lib,
+  system,
+  ...
+}:
+{
+  imports = lib.map (x: ./. + "/${x}") (
+    builtins.attrNames (
+      lib.removeAttrs (builtins.readDir ./.) [
+        "base.nix"
+        "all.nix"
+      ]
+    )
+  );
+
+  environment.systemPackages = lib.attrValues self.packages."${system}";
+}
diff --git a/machines/modules/base.nix b/machines/modules/base.nix
new file mode 100644
index 00000000..3d1fba1f
--- /dev/null
+++ b/machines/modules/base.nix
@@ -0,0 +1,15 @@
+{ system, lib, ... }:
+{
+  nixpkgs.system = system;
+  fileSystems = {
+    "/".device = lib.mkDefault "/dev/sda";
+  };
+  boot.loader.grub.devices = lib.mkDefault [ "/dev/sda" ];
+  virtualisation.vmVariant = {
+    virtualization = {
+      diskSize = 10 * 1024 * 1024 * 1024; # 10GB
+      memorySize = 8192; # 8GB
+      cores = 4;
+    };
+  };
+}
diff --git a/machines/modules/ethereum-validators-monitoring.nix b/machines/modules/ethereum-validators-monitoring.nix
new file mode 100644
index 00000000..98f5c0c7
--- /dev/null
+++ b/machines/modules/ethereum-validators-monitoring.nix
@@ -0,0 +1,20 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.ethereum-validators-monitoring
+  ];
+
+  services.ethereum-validators-monitoring = {
+    db = {
+      host = "http://localhost:8123/";
+      user = "ethereum";
+      password-file = ../blankpass.txt;
+      name = "ethereum";
+    };
+    instances = {
+      # The Ethereum Validator Monitoring sends out too many requests to servers,
+      # which causes attestations to be missing. Therefore, as we lack a
+      # dedicated server, we are unable to have e-v-m.
+    };
+  };
+}
diff --git a/machines/modules/folder-size-metrics.nix b/machines/modules/folder-size-metrics.nix
new file mode 100644
index 00000000..ee37c11b
--- /dev/null
+++ b/machines/modules/folder-size-metrics.nix
@@ -0,0 +1,15 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.folder-size-metrics
+  ];
+
+  services.folder-size-metrics = {
+    enable = true;
+    # args = { #Unchanged
+    #   port = 8888;
+    #   base-path = "/var/lib";
+    #   interval-sec = 60;
+    # };
+  };
+}
diff --git a/machines/modules/healthcheck.nix b/machines/modules/healthcheck.nix
new file mode 100644
index 00000000..90b28fa7
--- /dev/null
+++ b/machines/modules/healthcheck.nix
@@ -0,0 +1,53 @@
+{ self, pkgs, ... }:
+{
+  imports = [
+    self.modules.nixos.healthcheck
+  ];
+
+  systemd.services.test = {
+    description = "";
+    enable = true;
+    path = with pkgs; [
+    ];
+    serviceConfig = {
+      Type = "oneshot";
+      Restart = "on-failure";
+      RestartSec = "10s";
+
+      ExecStart = ''
+        sleep 10; # Simulate a startup delay.
+        echo "Starting test service...";
+        echo "Test" > /tmp/test.log
+        while true ; do
+          echo "Test" | nc -l 8080
+        done
+      '';
+    };
+  };
+
+  # --- Health Check Configuration ---
+  mcl.services.test.healthcheck = {
+    runtimePackages = with pkgs; [
+      netcat
+      curl
+    ];
+
+    # READINESS: Use the notify pattern to signal when the service is truly ready.
+    readiness-probe = {
+      enable = true;
+      command = "ls /tmp/test.log";
+      interval = 2;
+      statusWaitingMessage = "Test starting, waiting...";
+      statusReadyMessage = "Test is ready.";
+    };
+
+    # LIVENESS: After startup, use a timer to periodically check health.
+    liveness-probe = {
+      enable = true;
+      command = "[ \"$(nc -w 2 localhost 8080)\" = \"Test2\" ]";
+      initialDelay = 15;
+      interval = 30; # Check every 30 seconds.
+      timeout = 5;
+    };
+  };
+}
diff --git a/machines/modules/lido-keys-api.nix b/machines/modules/lido-keys-api.nix
new file mode 100644
index 00000000..5c3d0dac
--- /dev/null
+++ b/machines/modules/lido-keys-api.nix
@@ -0,0 +1,30 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-keys-api
+  ];
+
+  services.lido-keys-api = {
+    enable = true;
+    args = {
+      port = 3000;
+      cors-whitelist-regexp = "^https?://(?:.+?\.)?(?:lido|testnet|mainnet|holesky)\.fi$";
+      global-throttle-ttl = 5;
+      global-throttle-limit = 100;
+      global-cache-ttl = 1;
+      sentry-dsn = "";
+      log-level = "debug";
+      log-format = "json";
+      db-name = "node_operator_keys_service_db";
+      db-port = 5432;
+      db-host = "127.0.0.1";
+      db-user = "postgres";
+      db-password = ../blankpass.txt;
+      provider-json-rpc-max-batch-size = 100;
+      provider-concurrent-requests = 5;
+      provider-batch-aggregation-wait-ms = 10;
+      validator-registry-enable = true;
+    };
+  };
+
+}
diff --git a/machines/modules/lido-validator-ejector.nix b/machines/modules/lido-validator-ejector.nix
new file mode 100644
index 00000000..6aef8052
--- /dev/null
+++ b/machines/modules/lido-validator-ejector.nix
@@ -0,0 +1,25 @@
+{ self, config, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-validator-ejector
+  ];
+
+  services.lido-validator-ejector = {
+    enable = true;
+    args = {
+      messages-location = config.services.lido-withdrawals-automation.args.output-folder;
+      blocks-preload = 100000;
+      http-port = 8989;
+      run-metrics = true;
+      run-health-check = true;
+      logger-level = "info";
+      logger-format = "simple";
+      logger-secrets = [
+        "MESSAGES_PASSWORD"
+        "EXECUTION_NODE"
+        "CONSENSUS_NODE"
+      ];
+      dry-run = false;
+    };
+  };
+}
diff --git a/machines/modules/lido-withdrawals-automation.nix b/machines/modules/lido-withdrawals-automation.nix
new file mode 100644
index 00000000..7c5095e4
--- /dev/null
+++ b/machines/modules/lido-withdrawals-automation.nix
@@ -0,0 +1,16 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.lido-withdrawals-automation
+  ];
+
+  services.lido-withdrawals-automation = {
+    enable = true;
+    args = {
+      password = "";
+      percentage = 10;
+      output-folder = "/ethereum/lido/withdrawal-automation";
+      overwrite = "always";
+    };
+  };
+}
diff --git a/machines/modules/mcl-disko.nix b/machines/modules/mcl-disko.nix
new file mode 100644
index 00000000..f34f4703
--- /dev/null
+++ b/machines/modules/mcl-disko.nix
@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-disko
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}
diff --git a/machines/modules/mcl-host-info.nix b/machines/modules/mcl-host-info.nix
new file mode 100644
index 00000000..0d4cdceb
--- /dev/null
+++ b/machines/modules/mcl-host-info.nix
@@ -0,0 +1,12 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-host-info
+  ];
+
+  mcl.host-info = {
+    type = "server";
+    isDebugVM = true;
+    configPath = ./.;
+  };
+}
diff --git a/machines/modules/mcl-secrets.nix b/machines/modules/mcl-secrets.nix
new file mode 100644
index 00000000..0db51c68
--- /dev/null
+++ b/machines/modules/mcl-secrets.nix
@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.mcl-secrets
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}
diff --git a/machines/modules/pharos.nix b/machines/modules/pharos.nix
new file mode 100644
index 00000000..d32367ec
--- /dev/null
+++ b/machines/modules/pharos.nix
@@ -0,0 +1,11 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.pharos
+  ];
+
+  services.pharos = {
+    enable = true;
+    network = "testnet";
+  };
+}
diff --git a/machines/modules/pyroscope.nix b/machines/modules/pyroscope.nix
new file mode 100644
index 00000000..39af965f
--- /dev/null
+++ b/machines/modules/pyroscope.nix
@@ -0,0 +1,10 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.pyroscope
+  ];
+
+  services.pyroscope = {
+    enable = true;
+  };
+}
diff --git a/machines/modules/random-alerts.nix b/machines/modules/random-alerts.nix
new file mode 100644
index 00000000..e71cf17f
--- /dev/null
+++ b/machines/modules/random-alerts.nix
@@ -0,0 +1,8 @@
+{ self, ... }:
+{
+  imports = [
+    self.modules.nixos.random-alerts
+  ];
+
+  #TODO: Figure out the arguments for this service and enable it
+}

From 785c7422fdc2999b3ca55e9f2d9ac3587d3aa1c5 Mon Sep 17 00:00:00 2001
From: Simeon Armenchev <monyarm@gmail.com>
Date: Fri, 11 Jul 2025 17:05:10 +0300
Subject: [PATCH 3/4] wip

---
 checks/default.nix                            |  1 +
 checks/vm-tests.nix                           | 38 +++++++++++++++++++
 machines/default.nix                          |  7 ++++
 machines/modules/healthcheck.nix              |  2 +-
 machines/modules/lido-keys-api.nix            |  2 +-
 machines/modules/lido-validator-ejector.nix   |  3 ++
 .../modules/lido-withdrawals-automation.nix   |  1 +
 7 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 checks/vm-tests.nix

diff --git a/checks/default.nix b/checks/default.nix
index 0379200d..78ab1e53 100644
--- a/checks/default.nix
+++ b/checks/default.nix
@@ -3,5 +3,6 @@
   imports = [
     ./packages-ci-matrix.nix
     ./pre-commit.nix
+    ./vm-tests.nix
   ];
 }
diff --git a/checks/vm-tests.nix b/checks/vm-tests.nix
new file mode 100644
index 00000000..1fec8f75
--- /dev/null
+++ b/checks/vm-tests.nix
@@ -0,0 +1,38 @@
+{
+  lib,
+  inputs,
+  self,
+  ...
+}:
+{
+  perSystem =
+    {
+      inputs',
+      self',
+      pkgs,
+      ...
+    }:
+    {
+      checks = {
+        "healthcheck-test-01" = pkgs.testers.runNixOSTest {
+          name = "healthcheck-test-01";
+          nodes = {
+            machine =
+              { config, pkgs, ... }:
+              {
+
+                imports = [
+                  self.modules.nixos.machine_healthcheck
+                ];
+
+              };
+            testScript =
+              { nodes, ... }:
+              ''
+                machine.wait_for_unit("default.target")
+              '';
+          };
+        };
+      };
+    };
+}
diff --git a/machines/default.nix b/machines/default.nix
index a83b570d..2653defe 100644
--- a/machines/default.nix
+++ b/machines/default.nix
@@ -19,4 +19,11 @@ in
   flake.nixosConfigurations = (
     lib.mergeAttrsList (lib.map (x: mkMachine x) (builtins.attrNames (builtins.readDir ./modules)))
   );
+  flake.modules.nixos = (
+    lib.mergeAttrsList (
+      lib.map (x: {
+        "machine_${lib.removeSuffix ".nix" x}" = (import (./modules + "/${x}"));
+      }) (builtins.attrNames (builtins.readDir ./modules))
+    )
+  );
 }
diff --git a/machines/modules/healthcheck.nix b/machines/modules/healthcheck.nix
index 90b28fa7..47979bfb 100644
--- a/machines/modules/healthcheck.nix
+++ b/machines/modules/healthcheck.nix
@@ -10,7 +10,7 @@
     path = with pkgs; [
     ];
     serviceConfig = {
-      Type = "oneshot";
+      Type = "simple";
       Restart = "on-failure";
       RestartSec = "10s";
 
diff --git a/machines/modules/lido-keys-api.nix b/machines/modules/lido-keys-api.nix
index 5c3d0dac..c014cc16 100644
--- a/machines/modules/lido-keys-api.nix
+++ b/machines/modules/lido-keys-api.nix
@@ -19,7 +19,7 @@
       db-port = 5432;
       db-host = "127.0.0.1";
       db-user = "postgres";
-      db-password = ../blankpass.txt;
+      db-password = "";
       provider-json-rpc-max-batch-size = 100;
       provider-concurrent-requests = 5;
       provider-batch-aggregation-wait-ms = 10;
diff --git a/machines/modules/lido-validator-ejector.nix b/machines/modules/lido-validator-ejector.nix
index 6aef8052..6f63b48b 100644
--- a/machines/modules/lido-validator-ejector.nix
+++ b/machines/modules/lido-validator-ejector.nix
@@ -8,6 +8,9 @@
     enable = true;
     args = {
       messages-location = config.services.lido-withdrawals-automation.args.output-folder;
+      consensus-node = "";
+      execution-node = "";
+      locator-address = "";
       blocks-preload = 100000;
       http-port = 8989;
       run-metrics = true;
diff --git a/machines/modules/lido-withdrawals-automation.nix b/machines/modules/lido-withdrawals-automation.nix
index 7c5095e4..2f392749 100644
--- a/machines/modules/lido-withdrawals-automation.nix
+++ b/machines/modules/lido-withdrawals-automation.nix
@@ -7,6 +7,7 @@
   services.lido-withdrawals-automation = {
     enable = true;
     args = {
+      operator-id = "";
       password = "";
       percentage = 10;
       output-folder = "/ethereum/lido/withdrawal-automation";

From 9911f411c8234e44240a78c13a4f7583c21e6cf9 Mon Sep 17 00:00:00 2001
From: Simeon Armenchev <monyarm@gmail.com>
Date: Fri, 25 Jul 2025 17:16:28 +0300
Subject: [PATCH 4/4] wip

---
 checks/vm-tests.nix              | 31 ++++++++++++++++---------------
 machines/default.nix             | 16 +++++++++++++++-
 machines/modules/base.nix        |  3 +--
 machines/modules/healthcheck.nix |  2 +-
 4 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/checks/vm-tests.nix b/checks/vm-tests.nix
index 1fec8f75..6de756cf 100644
--- a/checks/vm-tests.nix
+++ b/checks/vm-tests.nix
@@ -10,28 +10,29 @@
       inputs',
       self',
       pkgs,
+      system,
       ...
     }:
     {
       checks = {
         "healthcheck-test-01" = pkgs.testers.runNixOSTest {
           name = "healthcheck-test-01";
-          nodes = {
-            machine =
-              { config, pkgs, ... }:
-              {
-
-                imports = [
-                  self.modules.nixos.machine_healthcheck
-                ];
-
-              };
-            testScript =
-              { nodes, ... }:
-              ''
-                machine.wait_for_unit("default.target")
-              '';
+          node.specialArgs = {
+            inherit self system lib;
           };
+          testScript = ''
+            machine.start()
+            machine.wait_for_unit("default.target")
+          '';
+          nodes.machine =
+            { pkgs, ... }:
+            {
+              imports = [
+                ../machines/modules/base.nix
+                ../machines/modules/healthcheck.nix
+              ];
+            };
+
         };
       };
     };
diff --git a/machines/default.nix b/machines/default.nix
index 2653defe..b2d1091c 100644
--- a/machines/default.nix
+++ b/machines/default.nix
@@ -5,10 +5,17 @@
 }:
 let
   system = "x86_64-linux";
+  pkgs = import self.inputs.nixpkgs {
+    inherit system;
+    config.allowUnfree = true;
+  };
   mkMachine = x: {
     "${lib.removeSuffix ".nix" x}" = lib.nixosSystem {
       specialArgs = { inherit self system; };
       modules = lib.unique [
+        {
+          nixpkgs.system = system;
+        }
         ./modules/base.nix
         ./modules/${x}
       ];
@@ -22,7 +29,14 @@ in
   flake.modules.nixos = (
     lib.mergeAttrsList (
       lib.map (x: {
-        "machine_${lib.removeSuffix ".nix" x}" = (import (./modules + "/${x}"));
+        "machine_${lib.removeSuffix ".nix" x}" = (
+          import (./modules + "/${x}") {
+            inherit
+              self
+              pkgs
+              ;
+          }
+        );
       }) (builtins.attrNames (builtins.readDir ./modules))
     )
   );
diff --git a/machines/modules/base.nix b/machines/modules/base.nix
index 3d1fba1f..757d5cd5 100644
--- a/machines/modules/base.nix
+++ b/machines/modules/base.nix
@@ -1,6 +1,5 @@
-{ system, lib, ... }:
+{ lib, ... }:
 {
-  nixpkgs.system = system;
   fileSystems = {
     "/".device = lib.mkDefault "/dev/sda";
   };
diff --git a/machines/modules/healthcheck.nix b/machines/modules/healthcheck.nix
index 47979bfb..c9eac97b 100644
--- a/machines/modules/healthcheck.nix
+++ b/machines/modules/healthcheck.nix
@@ -7,7 +7,7 @@
   systemd.services.test = {
     description = "";
     enable = true;
-    path = with pkgs; [
+    path = [
     ];
     serviceConfig = {
       Type = "simple";