diff --git a/nixos/modules/contracts/default.nix b/nixos/modules/contracts/default.nix new file mode 100644 index 0000000000000..172ca6743d3ac --- /dev/null +++ b/nixos/modules/contracts/default.nix @@ -0,0 +1,164 @@ +{ lib, ... }: +let + inherit (lib) mkOption; + inherit (lib.types) + attrs + lazyAttrsOf + functionTo + submodule + listOf + str + deferredModule + optionType + ; +in +{ + options.contracts = mkOption { + description = '' + Base option for a contract. + + To create a new contract, add an instance of `config.contracts.` + and define the `meta`, `input` and `output` options. + The `consumer` and `provider` options will then be set up automatically + and contain respectively the type of a consumer and provider + of this new contract. + + To use the `` contract, declare an option with either the + `config.contracts..consumer` or `config.contracts..provider` + type. + + The `behaviorTest` option is used to ensure all `provider` of a contract + behave the same way. + ''; + type = lazyAttrsOf ( + submodule (interface: { + options = { + meta = mkOption { + description = '' + Useful information about the contract and its maintenance. + ''; + type = submodule { + options = { + maintainers = mkOption { + description = '' + Maintainers of the contract. + ''; + type = listOf str; + }; + description = mkOption { + description = '' + Description of the contract. + ''; + type = str; + }; + }; + }; + }; + input = mkOption { + description = '' + Input type of a contract. + ''; + type = deferredModule; + }; + output = mkOption { + description = '' + Output type of a contract. + ''; + type = deferredModule; + }; + consumer = mkOption { + description = '' + Consumer type for a contract. + + This option is set up automatically. + Define instead the `input` and `output` options. + ''; + type = optionType; + readOnly = true; + defaultText = lib.literalExpression '' + submodule (consumer: { + options = { + provider = mkOption { + type = interface.config.provider; + }; + input = mkOption { + type = submodule interface.config.input; + }; + output = mkOption { + type = submodule interface.config.output; + readOnly = true; + default = consumer.config.provider.output; + }; + }; + }) + ''; + default = submodule (consumer: { + options = { + provider = mkOption { + type = interface.config.provider; + }; + input = mkOption { + type = submodule interface.config.input; + }; + output = mkOption { + type = submodule interface.config.output; + readOnly = true; + default = consumer.config.provider.output; + }; + }; + }); + }; + provider = mkOption { + description = '' + Provider type for a contract. + + This option is set up automatically. + Define instead the `input` and `output` options. + ''; + type = optionType; + readOnly = true; + defaultText = lib.literalExpression '' + submodule (provider: { + options = { + consumer = mkOption { + type = lib.types.nullOr interface.config.consumer; + default = null; + }; + input = mkOption { + type = lib.types.nullOr (submodule interface.config.input); + readOnly = true; + default = provider.config.consumer.input or null; + }; + output = mkOption { + type = submodule interface.config.output; + }; + }; + } + ''; + default = submodule (provider: { + options = { + consumer = mkOption { + type = interface.config.consumer; + }; + input = mkOption { + type = submodule interface.config.input; + readOnly = true; + default = provider.config.consumer.input; + }; + output = mkOption { + type = submodule interface.config.output; + }; + }; + }); + behaviorTest = mkOption { + # The type should be more precise of course. + # There should actually be a NixOSTest type. + # And we can probably do something fancy with the `input` and `output` deferred modules. + type = functionTo attrs; + }; + }; + }; + }) + ); + }; +} diff --git a/nixos/modules/contracts/fileBackup.nix b/nixos/modules/contracts/fileBackup.nix new file mode 100644 index 0000000000000..0717f1b86395a --- /dev/null +++ b/nixos/modules/contracts/fileBackup.nix @@ -0,0 +1,225 @@ +{ lib, ... }: +let + inherit (lib) mkOption; + inherit (lib.types) + str + path + nonEmptyListOf + listOf + submodule + ; +in +{ + contracts.fileBackup = { + meta = { + maintainers = [ lib.maintainers.ibizaman ]; + description = '' + File backup contract where a directory containing + regular files is to be backed up. + ''; + }; + + input = { + options.user = mkOption { + description = '' + Unix user doing the backups. + ''; + type = str; + example = "vaultwarden"; + }; + + options.sourceDirectories = mkOption { + description = "Directories to back up."; + type = nonEmptyListOf str; + example = "/var/lib/vaultwarden"; + }; + + options.excludePatterns = mkOption { + description = "File patterns to exclude."; + type = listOf str; + default = [ ]; + }; + + options.hooks = mkOption { + description = "Hooks to run around the backup."; + default = { }; + type = submodule { + options = { + beforeBackup = mkOption { + description = "Hooks to run before backup."; + type = listOf path; + default = [ ]; + }; + + afterBackup = mkOption { + description = "Hooks to run after backup."; + type = listOf path; + default = [ ]; + }; + }; + }; + }; + }; + + output = output: { + options.restoreScript = mkOption { + description = '' + Name of script that can restore the database. + One can then list snapshots with: + + ```bash + $ ${output.options.restoreScript.value} snapshots + ``` + + And restore the database with: + + ```bash + $ ${output.options.restoreScript.value} restore latest + ``` + ''; + type = path; + }; + + options.backupService = mkOption { + description = '' + Name of service backing up the database. + + This script can be ran manually to back up the database: + + ```bash + $ systemctl start ${output.options.backupService.value} + ``` + ''; + type = str; + }; + }; + + behaviorTest = + { + providerRoot, + extraModules ? [ ], + }: + { + nodes.machine = + { config, ... }: + { + imports = extraModules; + + options.test = { + repository = mkOption { + type = str; + default = "/opt/repository"; + }; + username = mkOption { + type = str; + default = "me"; + }; + sourceDirectories = mkOption { + type = listOf str; + default = [ + "/opt/files/A" + "/opt/files/B" + ]; + }; + }; + + config = lib.mkMerge [ + (lib.setAttrByPath providerRoot { + consumer.input = { + inherit (config.test) sourceDirectories; + user = config.test.username; + }; + }) + (lib.mkIf (config.test.username != "root") { + users.users.${config.test.username} = { + isSystemUser = true; + group = config.test.username; + }; + users.groups.${config.test.username} = { }; + }) + ]; + }; + + extraPythonPackages = p: [ p.dictdiffer ]; + + testScript = + { nodes, ... }: + let + cfg = nodes.machine; + inherit (lib.getAttrFromPath providerRoot nodes.machine) output; + in + '' + from dictdiffer import diff # type: ignore + + username = "${cfg.test.username}" + sourceDirectories = [ ${lib.concatMapStringsSep ", " (x: ''"${x}"'') cfg.test.sourceDirectories} ] + + def list_files(dir): + files_and_content = {} + + files = machine.succeed(f"""find {dir} -type f""").split("\n")[:-1] + + for f in files: + content = machine.succeed(f"""cat {f}""").strip() + files_and_content[f] = content + + return files_and_content + + def assert_files(dir, files): + result = list(diff(list_files(dir), files)) + if len(result) > 0: + raise Exception("Unexpected files:", result) + + with subtest("Create initial content"): + for path in sourceDirectories: + machine.succeed(f""" + mkdir -p {path} + echo repo_fileA_1 > {path}/fileA + echo repo_fileB_1 > {path}/fileB + + chown {username}: -R {path} + chmod go-rwx -R {path} + """) + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + + with subtest("First backup in repo"): + print(machine.succeed("systemctl cat ${output.backupService}")) + machine.succeed("systemctl start ${output.backupService}") + + with subtest("New content"): + for path in sourceDirectories: + machine.succeed(f""" + echo repo_fileA_2 > {path}/fileA + echo repo_fileB_2 > {path}/fileB + """) + + assert_files(path, { + f'{path}/fileA': 'repo_fileA_2', + f'{path}/fileB': 'repo_fileB_2', + }) + + with subtest("Delete content"): + for path in sourceDirectories: + machine.succeed(f"""rm -r {path}/*""") + + assert_files(path, {}) + + with subtest("Restore initial content from repo"): + machine.succeed("""${output.restoreScript} restore latest""") + + for path in sourceDirectories: + assert_files(path, { + f'{path}/fileA': 'repo_fileA_1', + f'{path}/fileB': 'repo_fileB_1', + }) + ''; + }; + }; + + meta.buildDocsInSandbox = false; +} diff --git a/nixos/modules/contracts/secret.nix b/nixos/modules/contracts/secret.nix new file mode 100644 index 0000000000000..39597d5ab271a --- /dev/null +++ b/nixos/modules/contracts/secret.nix @@ -0,0 +1,130 @@ +{ lib, ... }: +let + inherit (lib) mkOption; + inherit (lib.types) str path; +in +{ + contracts.secret = { + meta = { + maintainers = [ lib.maintainers.ibizaman ]; + description = '' + Contract for secrets handling where a consumer requests a secret + and a provider provides it at runtime at a given file path. + ''; + }; + + input = { + options.mode = mkOption { + description = '' + Mode of the secret file. + ''; + type = str; + }; + + options.owner = mkOption { + description = '' + Linux user owning the secret file. + ''; + type = str; + }; + + options.group = mkOption { + description = '' + Linux group owning the secret file. + ''; + type = str; + }; + }; + + output = { + options.path = mkOption { + type = path; + description = '' + Path to the file containing the secret generated out of band. + + This path will exist after deploying to a target host, + it is not available through the nix store. + ''; + }; + }; + + behaviorTest = + { + providerRoot, + extraModules ? [ ], + }: + { + nodes.machine = + { config, ... }: + { + imports = extraModules; + + options.test = { + owner = mkOption { + type = str; + default = "root"; + }; + + group = mkOption { + type = str; + default = "root"; + }; + + mode = mkOption { + type = str; + default = "0400"; + }; + + content = mkOption { + type = str; + default = "a super secret secret!"; + }; + }; + + config = lib.mkMerge [ + (lib.setAttrByPath providerRoot { + # We set consumer.input and not input directly because the latter is readOnly. + consumer.input = { + inherit (config.test) owner group mode; + }; + }) + (lib.mkIf (config.test.owner != "root") { + users.users.${config.test.owner}.isNormalUser = true; + }) + (lib.mkIf (config.test.group != "root") { + users.groups.${config.test.group} = { }; + }) + ]; + }; + + testScript = + { nodes, ... }: + let + cfg = nodes.machine; + inherit (lib.getAttrFromPath providerRoot nodes.machine) output; + in + '' + owner = machine.succeed("stat -c '%U' ${output.path}").strip() + print(f"Got owner {owner}") + if owner != "${cfg.test.owner}": + raise Exception(f"Owner should be '${cfg.test.owner}' but got '{owner}'") + + group = machine.succeed("stat -c '%G' ${output.path}").strip() + print(f"Got group {group}") + if group != "${cfg.test.group}": + raise Exception(f"Group should be '${cfg.test.group}' but got '{group}'") + + mode = str(int(machine.succeed("stat -c '%a' ${output.path}").strip())) + print(f"Got mode {mode}") + wantedMode = str(int("${cfg.test.mode}")) + if mode != wantedMode: + raise Exception(f"Mode should be '{wantedMode}' but got '{mode}'") + + content = machine.succeed("cat ${output.path}").strip() + print(f"Got content {content}") + if content != "${cfg.test.content}": + raise Exception(f"Content should be '${cfg.test.content}' but got '{content}'") + ''; + }; + }; +} diff --git a/nixos/modules/contracts/streamingBackup.nix b/nixos/modules/contracts/streamingBackup.nix new file mode 100644 index 0000000000000..c185f3cf1795a --- /dev/null +++ b/nixos/modules/contracts/streamingBackup.nix @@ -0,0 +1,193 @@ +{ lib, ... }: +let + inherit (lib) mkOption literalExpression; + inherit (lib.types) path str; +in +{ + contracts.streamingBackup = { + meta = { + maintainers = [ lib.maintainers.ibizaman ]; + description = '' + Streaming backup contract where what items to back up come from a stream. + + For example, this contract is well suited to back up a database or a tar file. + ''; + }; + + input = { + options = { + backupName = mkOption { + description = "Name of the backup in the repository."; + type = str; + example = "postgresql.sql"; + }; + + backupCmd = mkOption { + description = "A bash command that produces the database dump on stdout."; + type = str; + example = literalExpression '' + ''${pkgs.postgresql}/bin/pg_dumpall | ''${pkgs.gzip}/bin/gzip --rsyncable + ''; + }; + + restoreCmd = mkOption { + description = '' + A bash command that reads the database dump on stdin and restores the database. + ''; + type = str; + example = literalExpression '' + ''${pkgs.gzip}/bin/gunzip | ''${pkgs.postgresql}/bin/psql postgres + ''; + }; + }; + }; + + output = output: { + options = { + restoreScript = mkOption { + description = '' + Name of script that can restore the database. + One can then list snapshots with: + + ```bash + $ ${output.options.restoreScript.value} snapshots + ``` + + And restore the database with: + + ```bash + $ ${output.options.restoreScript.value} restore latest + ``` + ''; + type = path; + }; + + backupService = mkOption { + description = '' + Name of service backing up the database. + + This script can be ran manually to back up the database: + + ```bash + $ systemctl start ${(lib.debug.traceValFn lib.attrNames output.config).backupService.value} + ``` + ''; + type = str; + }; + }; + }; + + behaviorTest = + { + providerRoot, + extraModules ? [ ], + }: + { + nodes.machine = + { config, ... }: + { + imports = extraModules; + + options.test = { + repository = mkOption { + type = str; + default = "/opt/repository"; + }; + username = mkOption { + type = str; + default = "me"; + }; + backupName = mkOption { + type = str; + default = "db.bck"; + }; + }; + + config = lib.mkMerge [ + (lib.setAttrByPath providerRoot { + consumer = config.services.postgresql.streamingBackup; + }) + { + services.postgresql = { + streamingBackup.provider = lib.getAttrFromPath providerRoot; + + enable = true; + ensureDatabases = [ + config.test.username + "testdb" + ]; + ensureUsers = [ + { + name = config.test.username; + ensureDBOwnership = true; + ensureClauses.login = true; + } + ]; + }; + test.backupName = "db.bck"; + test.username = config.services.postgresql.superUser; + } + (lib.mkIf (config.test.username != "root") { + users.users.${config.test.username} = { + isSystemUser = true; + group = config.test.username; + }; + users.groups.${config.test.username} = { }; + }) + ]; + }; + + testScript = + { nodes, ... }: + let + cfg = nodes.machine; + inherit (lib.getAttrFromPath providerRoot nodes.machine) output; + in + '' + import csv + + start_all() + machine.wait_for_unit("postgresql.service") + machine.wait_for_open_port(5432) + + def peer_cmd(cmd, db="testdb"): + return "sudo -u ${cfg.test.username} psql -U ${cfg.test.username} {db} --csv --command \"{cmd}\"".format(cmd=cmd, db=db) + + def query(query): + res = machine.succeed(peer_cmd(query)) + return list(dict(l) for l in csv.DictReader(res.splitlines())) + + def cmp_tables(a, b): + for i in range(max(len(a), len(b))): + diff = set(a[i]) ^ set(b[i]) + if len(diff) > 0: + raise Exception(i, diff) + + table = [{'name': 'car', 'count': '1'}, {'name': 'lollipop', 'count': '2'}] + + with subtest("create fixture"): + machine.succeed(peer_cmd("CREATE TABLE test (name text, count int)")) + machine.succeed(peer_cmd("INSERT INTO test VALUES ('car', 1), ('lollipop', 2)")) + + res = query("SELECT * FROM test") + cmp_tables(res, table) + + with subtest("backup"): + print(machine.succeed("systemctl start ${output.backupService}")) + + with subtest("drop database"): + print(machine.succeed(peer_cmd("DROP DATABASE testdb", db="postgres"))) + machine.fail(peer_cmd("SELECT * FROM test")) + + with subtest("restore"): + print(machine.succeed("${output.restoreScript} restore latest")) + + with subtest("check restoration"): + res = query("SELECT * FROM test") + cmp_tables(res, table) + ''; + }; + }; + + meta.buildDocsInSandbox = false; +} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index f0beafd773641..2ea91fca9962d 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -48,6 +48,10 @@ ./config/xdg/sounds.nix ./config/xdg/terminal-exec.nix ./config/zram.nix + ./contracts/default.nix + ./contracts/fileBackup.nix + ./contracts/secret.nix + ./contracts/streamingBackup.nix ./hardware/acpilight.nix ./hardware/all-firmware.nix ./hardware/all-hardware.nix @@ -1941,6 +1945,7 @@ ./tasks/stratis.nix ./tasks/swraid.nix ./tasks/trackpoint.nix + ./testing/hardcodedSecret.nix ./testing/service-runner.nix ./virtualisation/amazon-options.nix ./virtualisation/appvm.nix diff --git a/nixos/modules/services/backup/restic.nix b/nixos/modules/services/backup/restic.nix index 85bb1cd0bf412..004354228734f 100644 --- a/nixos/modules/services/backup/restic.nix +++ b/nixos/modules/services/backup/restic.nix @@ -16,7 +16,7 @@ in ''; type = lib.types.attrsOf ( lib.types.submodule ( - { name, ... }: + mod@{ name, ... }: { options = { passwordFile = lib.mkOption { @@ -318,7 +318,91 @@ in ''; example = 0.1; }; + + fileBackup = lib.mkOption { + type = lib.types.nullOr config.contracts.fileBackup.provider; + default = null; + }; + streamingBackup = lib.mkOption { + type = lib.types.nullOr config.contracts.streamingBackup.provider; + default = null; + }; }; + + config = lib.mkMerge [ + (lib.mkIf (mod.config.fileBackup.input != null) ( + let + inherit (mod.config) fileBackup; + in + { + user = fileBackup.input.user; + paths = fileBackup.input.sourceDirectories; + backupPrepareCommand = lib.concatStringsSep "\n" fileBackup.input.hooks.beforeBackup; + backupCleanupCommand = lib.concatStringsSep "\n" fileBackup.input.hooks.afterBackup; + exclude = fileBackup.input.excludePatterns; + } + )) + { + fileBackup.output = { + backupService = "restic-backups-${name}.service"; + restoreScript = lib.getExe ( + pkgs.writeShellApplication { + name = "restic-${name}"; + text = '' + if [ "$1" = "snapshots" ]; then + restic-${name} snapshots + elif [ "$1" = "restore" ]; then + shift + restic-${name} restore "$1" --target / + fi + ''; + } + ); + }; + } + + (lib.mkIf (mod.config.streamingBackup.input != null) ( + let + inherit (mod.config.streamingBackup) input; + in + { + command = [ + (lib.getExe ( + pkgs.writeShellApplication { + name = "dump.sh"; + text = input.backupCmd; + } + )) + ]; + extraBackupArgs = [ + "--stdin-filename ${input.backupName}" + ]; + } + )) + ( + let + inherit (mod.config.streamingBackup) input; + in + { + streamingBackup.output = { + backupService = "restic-backups-${name}.service"; + restoreScript = lib.getExe ( + pkgs.writeShellApplication { + name = "restic-${name}"; + text = '' + if [ "$1" = "snapshots" ]; then + restic-${name} snapshots + elif [ "$1" = "restore" ]; then + shift + restic-${name} dump "$1" ${input.backupName} | ${input.restoreCmd} + fi + ''; + } + ); + }; + } + ) + ]; } ) ); diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index f4b309fbe4b63..0f90571cd39cd 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -627,6 +627,10 @@ in this value would lead to breakage while setting up databases. ''; }; + + streamingBackup = mkOption { + type = config.contracts.streamingBackup.consumer; + }; }; }; @@ -957,6 +961,18 @@ in ) cfg.ensureUsers} ''; }; + + services.postgresql.streamingBackup = { + input.backupName = "postgres.sql"; + + input.backupCmd = '' + ${pkgs.sudo}/bin/sudo -u ${cfg.superUser} ${pkgs.postgresql}/bin/pg_dumpall -U ${cfg.superUser} | ${pkgs.gzip}/bin/gzip --rsyncable + ''; + + input.restoreCmd = '' + ${pkgs.gzip}/bin/gunzip | ${pkgs.sudo}/bin/sudo -u ${cfg.superUser} ${pkgs.postgresql}/bin/psql postgres -U ${cfg.superUser} + ''; + }; }; meta.doc = ./postgresql.md; diff --git a/nixos/modules/services/web-apps/nextcloud.nix b/nixos/modules/services/web-apps/nextcloud.nix index 6e50d5e517a13..2bc8db41c88a7 100644 --- a/nixos/modules/services/web-apps/nextcloud.nix +++ b/nixos/modules/services/web-apps/nextcloud.nix @@ -1188,6 +1188,10 @@ in }; imaginary.enable = lib.mkEnableOption "Imaginary"; + + fileBackup = lib.mkOption { + type = config.contracts.fileBackup.consumer; + }; }; config = lib.mkIf cfg.enable ( @@ -1785,6 +1789,18 @@ in settings.return-size = true; }; } + + { + services.nextcloud.fileBackup.input = { + user = "nextcloud"; + sourceDirectories = [ + cfg.datadir + ]; + excludePatterns = [ + ".rnd" + ]; + }; + } ] ); diff --git a/nixos/modules/services/web-apps/stash.nix b/nixos/modules/services/web-apps/stash.nix index 72d4d67cd6cf9..8ca657ff328db 100644 --- a/nixos/modules/services/web-apps/stash.nix +++ b/nixos/modules/services/web-apps/stash.nix @@ -418,7 +418,7 @@ in }; passwordFile = mkOption { - type = types.nullOr types.path; + type = types.nullOr config.contracts.secret.consumer; default = null; example = "/path/to/password/file"; description = '' @@ -432,11 +432,11 @@ in }; jwtSecretKeyFile = mkOption { - type = types.path; + type = config.contracts.secret.consumer; description = "Path to file containing a secret used to sign JWT tokens."; }; sessionStoreKeyFile = mkOption { - type = types.path; + type = config.contracts.secret.consumer; description = "Path to file containing a secret for session store."; }; @@ -479,6 +479,26 @@ in scrapers_path = mkIf (!cfg.mutableScrapers) cfg.scrapers; }; + # I would prefer to have these in the "default" attr of the `services.stash.passwordFile` option + # but I couldn't make it work. + services.stash.passwordFile.input = mkIf (cfg.passwordFile != null) { + owner = cfg.user; + group = cfg.group; + mode = "0400"; + }; + # Just as an example, this is forbidden thanks to the readOnly property in consumer.output; + # services.stash.jwtSecretKeyFile.output.path = "/ash"; + services.stash.jwtSecretKeyFile.input = { + owner = cfg.user; + group = cfg.group; + mode = "0400"; + }; + services.stash.sessionStoreKeyFile.input = { + owner = cfg.user; + group = cfg.group; + mode = "0400"; + }; + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.settings.port ]; users.users.${cfg.user} = { @@ -514,9 +534,9 @@ in install -d ${cfg.settings.generated} if [[ -z "${toString cfg.mutableSettings}" || ! -f ${cfg.dataDir}/config.yml ]]; then env \ - password=$(< ${cfg.passwordFile}) \ - jwtSecretKeyFile=$(< ${cfg.jwtSecretKeyFile}) \ - sessionStoreKeyFile=$(< ${cfg.sessionStoreKeyFile}) \ + password=$(< ${cfg.passwordFile.output.path}) \ + jwtSecretKeyFile=$(< ${cfg.jwtSecretKeyFile.output.path}) \ + sessionStoreKeyFile=$(< ${cfg.sessionStoreKeyFile.output.path}) \ ${lib.getExe pkgs.yq-go} ' .jwt_secret_key = strenv(jwtSecretKeyFile) | .session_store_key = strenv(sessionStoreKeyFile) | diff --git a/nixos/modules/testing/hardcodedSecret.nix b/nixos/modules/testing/hardcodedSecret.nix new file mode 100644 index 0000000000000..c63619f6a00cf --- /dev/null +++ b/nixos/modules/testing/hardcodedSecret.nix @@ -0,0 +1,142 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.testing.hardcodedSecret; + + inherit (lib) mapAttrs' mkOption nameValuePair; + inherit (lib.types) + attrsOf + nullOr + str + submodule + ; + inherit (pkgs) writeText; +in +{ + options.testing.hardcodedSecret = mkOption { + default = { }; + description = '' + Hardcoded secrets. These should only be used in tests. + ''; + + example = lib.literalExpression '' + { + mySecret = { + secret.input = { + user = "me"; + mode = "0400"; + restartUnits = [ "myservice.service" ]; + }; + settings.content = "My Secret"; + }; + } + ''; + type = attrsOf ( + submodule ( + mod@{ name, options, ... }: + { + options = { + mode = mkOption { + description = '' + Mode of the secret file. + ''; + type = str; + default = "0400"; + }; + + owner = mkOption { + description = '' + Linux user owning the secret file. + ''; + type = str; + }; + + group = mkOption { + description = '' + Linux group owning the secret file. + ''; + type = str; + default = options.user.default; + defaultText = "user"; + }; + + content = mkOption { + type = nullOr str; + description = '' + Content of the secret as a string. + + This will be stored in the nix store and should only be used for testing or maybe in dev. + ''; + default = null; + }; + + source = mkOption { + type = nullOr str; + description = '' + Source of the content of the secret as a path in the nix store. + ''; + default = null; + }; + + path = mkOption { + type = str; + description = '' + Path where the secret should be located. + ''; + default = "/run/hardcodedSecrets/hardcodedSecret_${name}"; + }; + + secret = mkOption { + type = config.contracts.secret.provider; + }; + }; + + config = { + inherit (mod.config.secret.input) mode owner group; + secret.output.path = mod.config.path; + }; + } + ) + ); + }; + + config = { + system.activationScripts = mapAttrs' ( + n: cfg': + let + source = + if cfg'.source != null then cfg'.source else writeText "hardcodedSecret_${n}_content" cfg'.content; + in + nameValuePair "hardcodedSecret_${n}" '' + ( + set -e + mkdir -p "$(dirname "${cfg'.path}")" + touch "${cfg'.path}" + chmod ${cfg'.mode} "${cfg'.path}" + chown ${cfg'.owner}:${cfg'.group} "${cfg'.path}" + cp ${source} "${cfg'.path}" + ) || echo "Failed to create hardcoded secret at ${cfg'.path}" + '' + ) cfg; + }; + + # Without `meta.buildDocsInSandbox = false;`, I get: + # + # > error: attribute 'contracts' missing + # > at /nix/store/2gd9yzcfpqqp00vskxlqq4ds48mpgdzv-nixos/modules/testing/hardcodedSecret.nix:81:18: + # > 80| secret = mkOption { + # > 81| type = config.contracts.secret.provider; + # > | ^ + # > 82| }; + # > Cacheable portion of option doc build failed. + # > Usually this means that an option attribute that ends up in documentation (eg `default` or `description`) depends on the restricted module arguments `config` or `pkgs`. + # > + # > Rebuild your configuration with `--show-trace` to find the offending location. Remove the references to restricted arguments (eg by escaping their antiquotations or adding a `defaultText`) or disable the sandboxed build for the failing module by setting `meta.buildDocsInSandbox = false`. + # + # With the line, I don't get the warning but still get the missing 'contracts' attribute error. + meta.buildDocsInSandbox = false; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 58cddfabe0b7f..3a5a25cbb2dfe 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -404,6 +404,9 @@ in containers-restart_networking = runTest ./containers-restart_networking.nix; containers-tmpfs = runTest ./containers-tmpfs.nix; containers-unified-hierarchy = runTest ./containers-unified-hierarchy.nix; + contracts-fileBackup-restic = runTest ./contracts/fileBackup/restic.nix; + contracts-secret-hardcodedSecret = runTest ./contracts/secret/hardcodedSecret.nix; + contracts-streamingBackup-restic = runTest ./contracts/streamingBackup/restic.nix; convos = runTest ./convos.nix; corerad = runTest ./corerad.nix; corteza = runTest ./corteza.nix; diff --git a/nixos/tests/contracts/fileBackup/restic.nix b/nixos/tests/contracts/fileBackup/restic.nix new file mode 100644 index 0000000000000..bc6a88e12a3c9 --- /dev/null +++ b/nixos/tests/contracts/fileBackup/restic.nix @@ -0,0 +1,51 @@ +{ + lib, + config, + pkgs, + ... +}: +let + inherit + ((import ../../../modules/contracts/fileBackup.nix { + inherit lib; + }).contracts.fileBackup + ) + behaviorTest + ; +in +{ + name = "contracts-fileBackup-restic"; + meta.maintainers = [ lib.maintainers.ibizaman ]; + # I tried using the following line but it leads to infinite recursion. + # Instead, I made a hacky import. pkgs.callPackage was also giving an + # infinite recursion. + # + # } // config.contracts.fileBackup.behaviorTest { + # + # Maybe the answer is in how [[file:~/Projects/nixpkgs/pkgs/development/cuda-modules/modules/generic/types/default.nix::config.generic.types = {][this]] works. +} +// behaviorTest { + providerRoot = [ + "services" + "restic" + "backups" + "mybackup" + "fileBackup" + ]; + extraModules = [ + ( + { config, ... }: + { + systemd.tmpfiles.rules = [ + "d '${config.test.repository}' 0750 ${config.test.username} root - -" + ]; + + services.restic.backups.mybackup = { + inherit (config.test) repository; + passwordFile = toString (pkgs.writeText "password" "password"); + initialize = true; + }; + } + ) + ]; +} diff --git a/nixos/tests/contracts/secret/hardcodedSecret.nix b/nixos/tests/contracts/secret/hardcodedSecret.nix new file mode 100644 index 0000000000000..8125f131b6f49 --- /dev/null +++ b/nixos/tests/contracts/secret/hardcodedSecret.nix @@ -0,0 +1,42 @@ +{ + lib, + config, + pkgs, + ... +}: +let + inherit + ((import ../../../modules/contracts/secret.nix { + inherit lib; + }).contracts.secret + ) + behaviorTest + ; +in +{ + name = "contracts-filebackup-restic"; + meta.maintainers = [ lib.maintainers.ibizaman ]; + # I tried using the following line but it leads to infinite recursion. + # Instead, I made a hacky import. pkgs.callPackage was also giving an + # infinite recursion. + # + # } // config.contracts.secret.behaviorTest { + # +} +// behaviorTest { + providerRoot = [ + "testing" + "hardcodedSecret" + "mysecret" + "secret" + ]; + extraModules = [ + ../../../modules/testing/hardcodedSecret.nix + ( + { config, ... }: + { + testing.hardcodedSecret.mysecret.content = config.test.content; + } + ) + ]; +} diff --git a/nixos/tests/contracts/streamingBackup/restic.nix b/nixos/tests/contracts/streamingBackup/restic.nix new file mode 100644 index 0000000000000..3d643a35798ed --- /dev/null +++ b/nixos/tests/contracts/streamingBackup/restic.nix @@ -0,0 +1,51 @@ +{ + lib, + config, + pkgs, + ... +}: +let + inherit + ((import ../../../modules/contracts/streamingBackup.nix { + inherit lib; + }).contracts.streamingBackup + ) + behaviorTest + ; +in +{ + name = "contracts-streamingBackup-postgresql"; + meta.maintainers = [ lib.maintainers.ibizaman ]; + # I tried using the following line but it leads to infinite recursion. + # Instead, I made a hacky import. pkgs.callPackage was also giving an + # infinite recursion. + # + # } // config.contracts.streamingBackup.behaviorTest { + # + # Maybe the answer is in how [[file:~/Projects/nixpkgs/pkgs/development/cuda-modules/modules/generic/types/default.nix::config.generic.types = {][this]] works. +} +// behaviorTest { + providerRoot = [ + "services" + "restic" + "backups" + "mybackup" + "streamingBackup" + ]; + extraModules = [ + ( + { config, ... }: + { + systemd.tmpfiles.rules = [ + "d '${config.test.repository}' 0750 ${config.test.username} root - -" + ]; + + services.restic.backups.mybackup = { + inherit (config.test) repository; + passwordFile = toString (pkgs.writeText "password" "password"); + initialize = true; + }; + } + ) + ]; +} diff --git a/nixos/tests/nextcloud/default.nix b/nixos/tests/nextcloud/default.nix index 72f7a28070b5c..a3961ad2e0c5a 100644 --- a/nixos/tests/nextcloud/default.nix +++ b/nixos/tests/nextcloud/default.nix @@ -54,7 +54,10 @@ let nodes = { client = { ... }: { }; nextcloud = - { lib, ... }: + let + topLevelConfig = config; + in + { lib, config, ... }: { networking.firewall.allowedTCPPorts = [ 80 ]; services.nextcloud = { @@ -63,8 +66,20 @@ let https = false; database.createLocally = lib.mkDefault true; config = { - adminpassFile = "${pkgs.writeText "adminpass" config.adminpass}"; # Don't try this at home! + adminpassFile = "${pkgs.writeText "adminpass" topLevelConfig.adminpass}"; # Don't try this at home! }; + fileBackup.provider = config.services.restic.backups.nextcloud.fileBackup; + }; + + systemd.tmpfiles.rules = [ + "d '/var/lib/backups/nextcloud' 0750 nextcloud root - -" + ]; + services.restic.backups.nextcloud = { + repository = "/var/lib/backups/nextcloud"; + passwordFile = toString (pkgs.writeText "password" "password"); + initialize = true; + + fileBackup.consumer = config.services.nextcloud.fileBackup; }; }; }; @@ -93,6 +108,9 @@ let "${test-helpers.rclone} ${test-helpers.check-sample}" ) + with subtest("Backup using file backup contract"): + nextcloud.succeed("systemctl start ${nodes.nextcloud.services.nextcloud.fileBackup.output.backupService}") + ${ if pkgs.lib.isFunction test-helpers.extraTests then test-helpers.extraTests args diff --git a/nixos/tests/stash.nix b/nixos/tests/stash.nix index 838a5e8a43c6d..d7611cadb27ae 100644 --- a/nixos/tests/stash.nix +++ b/nixos/tests/stash.nix @@ -9,61 +9,79 @@ import ./make-test-python.nix ( name = "stash"; meta.maintainers = pkgs.stash.meta.maintainers; - nodes.machine = { - services.stash = { - inherit dataDir; - enable = true; + nodes.machine = + { config, ... }: + { + imports = [ + ../modules/testing/hardcodedSecret.nix + ]; - username = "test"; - passwordFile = pkgs.writeText "stash-password" "MyPassword"; + services.stash = { + inherit dataDir; + enable = true; - jwtSecretKeyFile = pkgs.writeText "jwt_secret_key" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - sessionStoreKeyFile = pkgs.writeText "session_store_key" "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + username = "test"; + passwordFile.provider = config.testing.hardcodedSecret."stash-password".secret; - plugins = - let - src = pkgs.fetchFromGitHub { - owner = "stashapp"; - repo = "CommunityScripts"; - rev = "9b6fac4934c2fac2ef0859ea68ebee5111fc5be5"; - hash = "sha256-PO3J15vaA7SD4r/LyHlXjnpaeYAN9Q++O94bIWdz7OA="; - }; - in - [ - (pkgs.runCommand "stashNotes" { inherit src; } '' - mkdir -p $out/plugins - cp -r $src/plugins/stashNotes $out/plugins/stashNotes - '') - (pkgs.runCommand "Theme-Plex" { inherit src; } '' - mkdir -p $out/plugins - cp -r $src/themes/Theme-Plex $out/plugins/Theme-Plex - '') - ]; + jwtSecretKeyFile.provider = config.testing.hardcodedSecret."jwt_secret_key".secret; + sessionStoreKeyFile.provider = config.testing.hardcodedSecret."session_store_key".secret; - mutableScrapers = true; - scrapers = - let - src = pkgs.fetchFromGitHub { - owner = "stashapp"; - repo = "CommunityScrapers"; - rev = "2ece82d17ddb0952c16842b0775274bcda598d81"; - hash = "sha256-AEmnvM8Nikhue9LNF9dkbleYgabCvjKHtzFpMse4otM="; - }; - in - [ - (pkgs.runCommand "FTV" { inherit src; } '' - mkdir -p $out/scrapers/FTV - cp -r $src/scrapers/FTV.yml $out/scrapers/FTV - '') - ]; + plugins = + let + src = pkgs.fetchFromGitHub { + owner = "stashapp"; + repo = "CommunityScripts"; + rev = "9b6fac4934c2fac2ef0859ea68ebee5111fc5be5"; + hash = "sha256-PO3J15vaA7SD4r/LyHlXjnpaeYAN9Q++O94bIWdz7OA="; + }; + in + [ + (pkgs.runCommand "stashNotes" { inherit src; } '' + mkdir -p $out/plugins + cp -r $src/plugins/stashNotes $out/plugins/stashNotes + '') + (pkgs.runCommand "Theme-Plex" { inherit src; } '' + mkdir -p $out/plugins + cp -r $src/themes/Theme-Plex $out/plugins/Theme-Plex + '') + ]; - settings = { - inherit host port; + mutableScrapers = true; + scrapers = + let + src = pkgs.fetchFromGitHub { + owner = "stashapp"; + repo = "CommunityScrapers"; + rev = "2ece82d17ddb0952c16842b0775274bcda598d81"; + hash = "sha256-AEmnvM8Nikhue9LNF9dkbleYgabCvjKHtzFpMse4otM="; + }; + in + [ + (pkgs.runCommand "FTV" { inherit src; } '' + mkdir -p $out/scrapers/FTV + cp -r $src/scrapers/FTV.yml $out/scrapers/FTV + '') + ]; - stash = [ { path = "/srv"; } ]; + settings = { + inherit host port; + + stash = [ { path = "/srv"; } ]; + }; + }; + testing.hardcodedSecret."stash-password" = { + secret.consumer = config.services.stash.passwordFile; + content = "MyPassword"; + }; + testing.hardcodedSecret."jwt_secret_key" = { + secret.consumer = config.services.stash.jwtSecretKeyFile; + content = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + }; + testing.hardcodedSecret."session_store_key" = { + secret.consumer = config.services.stash.sessionStoreKeyFile; + content = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; }; }; - }; testScript = '' machine.wait_for_unit("stash.service")