diff --git a/hosts/blackberry/default.nix b/hosts/blackberry/default.nix index e2d8cd6f..a79019ae 100644 --- a/hosts/blackberry/default.nix +++ b/hosts/blackberry/default.nix @@ -33,7 +33,6 @@ backlight = false; # PC GPUs don't do that battery = false; # PC, doesn't have a battery }; - restic.usb-backups = true; impermanence.presets = { enable = true; diff --git a/hosts/blueberry/default.nix b/hosts/blueberry/default.nix index e6a6786a..4ddfe5e7 100644 --- a/hosts/blueberry/default.nix +++ b/hosts/blueberry/default.nix @@ -34,7 +34,6 @@ boot.kernelPackages = pkgs.linuxPackages; nixchad = { boot.bootloader = "systemd-boot"; - restic.usb-backups = true; smartctl-exporter.devices = [ "/dev/nvme0n1" ]; waybar.battery = true; impermanence = { diff --git a/nixosModules/common-services/default.nix b/nixosModules/common-services/default.nix index f90e179d..edd262c3 100644 --- a/nixosModules/common-services/default.nix +++ b/nixosModules/common-services/default.nix @@ -3,6 +3,6 @@ ./alloy ./smartctl-exporter.nix ./syncthing.nix - ./restic.nix + ./restic ]; } diff --git a/nixosModules/common-services/restic/default.nix b/nixosModules/common-services/restic/default.nix new file mode 100644 index 00000000..66044491 --- /dev/null +++ b/nixosModules/common-services/restic/default.nix @@ -0,0 +1,7 @@ +{ + imports = [ + ./module.nix + # ./legacy.nix + ./values.nix + ]; +} diff --git a/nixosModules/common-services/restic.nix b/nixosModules/common-services/restic/legacy.nix similarity index 100% rename from nixosModules/common-services/restic.nix rename to nixosModules/common-services/restic/legacy.nix diff --git a/nixosModules/common-services/restic/module.nix b/nixosModules/common-services/restic/module.nix new file mode 100644 index 00000000..79b02f89 --- /dev/null +++ b/nixosModules/common-services/restic/module.nix @@ -0,0 +1,402 @@ +{ + config, + lib, + pkgs, + utils, + ... +}: +let + # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers" + inherit (utils.systemdUtils.unitOptions) unitOption; +in +{ + options.nixchad.resticModule = { + enable = lib.mkEnableOption "restic" // { + default = true; + }; + + sources = lib.mkOption { + description = '' + Sources from which backups will be created. + ''; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + unitOptions = lib.mkOption { + description = "Additional systemd unit options"; + type = unitOption; + default = { }; + }; + + backupPre = lib.mkOption { + description = "Commands to run before the backup"; + type = lib.types.nullOr lib.types.str; + default = ""; + }; + + backupPost = lib.mkOption { + description = "Commands to run after the backup"; + type = lib.types.nullOr lib.types.str; + default = ""; + }; + + paths = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Which paths to backup, in addition to ones specified via + `dynamicFilesFrom`. If null or an empty array and + `dynamicFilesFrom` is also null, no backup command will be run. + This can be used to create a prune-only job. + ''; + example = [ + "/var/lib/postgresql" + "/home/user/backup" + ]; + }; + + command = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Command to pass to --stdin-from-command. If null or an empty array, and `paths`/`dynamicFilesFrom` + are also null, no backup command will be run. + ''; + example = [ + "sudo" + "-u" + "postgres" + "pg_dumpall" + ]; + }; + + exclude = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Patterns to exclude when backing up. See + https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for + details on syntax. + ''; + example = [ + "/var/cache" + "/home/*/.cache" + ".git" + ]; + }; + + dynamicFilesFrom = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + A script that produces a list of files to back up. The + results of this command are given to the '--files-from' + option. The result is merged with paths specified via `paths`. + ''; + example = "find /home/matt/git -type d -name .git"; + }; + + extraBackupArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Extra arguments passed to restic backup. + ''; + example = [ + "--cleanup-cache" + "--exclude-file=/etc/nixos/restic-ignore" + ]; + }; + }; + } + ); + }; + + destinations = lib.mkOption { + description = '' + Destinations where backups will be stored. + ''; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + init = lib.mkOption { + description = "Whether to init the destination repository prior to doing any backups"; + type = lib.types.bool; + default = true; + }; + + unitOptions = lib.mkOption { + description = "Additional systemd unit options"; + type = unitOption; + default = { }; + }; + + extraOptions = lib.mkOption { + description = "Additional options added to restic invokation via '-o' flag"; + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + + settings = lib.mkOption { + description = '' + Restic settings. They are provided as environment variables. They should be provided in upper snake + case (e.g. {env}`RESTIC_PASSWORD_FILE`). See + for supported options. + + This option can also take rclone settings, also as environment variables. They should be provided in + upper snake case (e.g. {env}`RCLONE_SKIP_LINKS`). See for supported + options. Restic will automatically supply the remote type and name for you. To provide secrets to the + backend, it's recommended to create rclone config file yourself, and use {env}`RCLONE_CONFIG` option + to point to it. It is also recommended to use a separate config file if you care about + case-sensitivity for your remote name. + ''; + type = lib.types.submodule { + freeformType = + with lib.types; + attrsOf (oneOf [ + str + (listOf str) + ]); + + options = { + RESTIC_REPOSITORY = lib.mkOption { + type = with lib.types; nullOr str; + default = null; + description = '' + repository to backup to. + ''; + example = "sftp:backup@192.168.1.100:/backups/my-backup"; + }; + RESTIC_REPOSITORY_FILE = lib.mkOption { + type = with lib.types; nullOr path; + default = null; + description = '' + Path to the file containing the repository location to backup to. + ''; + }; + RESTIC_PASSWORD_FILE = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Read the repository password from a file. + ''; + example = "/etc/nixos/restic-password"; + }; + + RCLONE_CONFIG = lib.mkOption { + type = + with lib.types; + nullOr (oneOf [ + str + path + ]); + default = null; + description = '' + Location of the rclone configuration file. + ''; + }; + }; + }; + example = lib.literalExpression '' + RESTIC_REPOSITORY = "s3:s3.us-east-1.amazonaws.com/bucket_name/restic"; + RESTIC_PASSWORD_FILE = "/secrets/password-file"; + AWS_ACCESS_KEY_ID = "XXXX"; + AWS_SECRET_ACCESS_KEY = "YYYY"; + RCLONE_BWLIMIT = "10M"; + RCLONE_HARD_DELETE = "true"; + # RCLONE_S3_PROVIDER = "AWS"; + # RCLONE_CONFIG_MYS3_ACCESS_KEY_ID = "XXXX"; + # RCLONE_CONFIG = "/my/config/file"; + ''; + }; + }; + } + ); + }; + + mappings = lib.mkOption { + description = '' + Mappings between sources and destinations. + ''; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + sources = lib.mkOption { + description = "Backup sources"; + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + destinations = lib.mkOption { + description = "Backup destinations"; + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + }; + } + ); + }; + }; + + config = + let + destinationTemplater = + name: destination: + let + extraOptions = lib.concatMapStrings (arg: " -o ${arg}") destination.extraOptions; + inhibitCmd = lib.concatStringsSep " " [ + "${pkgs.systemd}/bin/systemd-inhibit" + "--mode='block'" + "--who='restic'" + "--what='idle:sleep:shutdown:handle-lid-switch'" + "--why=${lib.escapeShellArg "Initializing destination ${name}"} " + ]; + resticCmd = "${inhibitCmd}${lib.getExe pkgs.restic}${extraOptions}"; + in + { + environment = destination.settings; + path = [ config.programs.ssh.package ]; + restartIfChanged = false; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "restic-destination-${name}" "${resticCmd} cat config > /dev/null || ${resticCmd} init"; + User = "root"; + Group = "root"; + RuntimeDirectory = "restic-destination-${name}"; + CacheDirectory = "restic-destination-${name}"; + CacheDirectoryMode = "0700"; + PrivateTmp = true; + # We don't want this to re-run needlessly + RemainAfterExit = true; + }; + }; + + # mapping is { source = { name = "foo"; ..}; ..} + backupTemplater = + mapping: + let + inherit (mapping) destination source; + + extraOptions = lib.concatMapStrings (arg: " -o ${arg}") destination.extraOptions; + inhibitCmd = lib.concatStringsSep " " [ + "${pkgs.systemd}/bin/systemd-inhibit" + "--mode='block'" + "--who='restic'" + "--what='idle:sleep:shutdown:handle-lid-switch'" + "--why=${lib.escapeShellArg "Scheduled backup ${source.name}"} " + ]; + resticCmd = "${inhibitCmd}${lib.getExe pkgs.restic}${extraOptions}"; + excludeFlags = lib.optional ( + source.exclude != [ ] + ) "--exclude-file=${pkgs.writeText "exclude-patterns" (lib.concatStringsSep "\n" source.exclude)}"; + fileBackup = (source.dynamicFilesFrom != null) || (source.paths != [ ]); + commandBackup = source.command != [ ]; + filesFromTmpFile = "/run/restic-backup-${source.name}-to-${destination.name}"; + + args = lib.concatStringsSep " " ( + source.extraBackupArgs + ++ lib.optionals fileBackup (excludeFlags ++ [ "--files-from=${filesFromTmpFile}" ]) + ++ lib.optionals commandBackup ([ "--stdin-from-command=true -- " ] ++ source.command) + ); + in + { + environment = destination.settings; + path = [ config.programs.ssh.package ]; + restartIfChanged = false; + wants = [ + "network-online.target" + "restic-destination-${destination.name}.service" + ]; + after = [ + "network-online.target" + "restic-destination-${destination.name}.service" + ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${resticCmd} backup ${args}"; + User = "root"; + Group = "root"; + RuntimeDirectory = "restic-backup-${source.name}-to-${destination.name}"; + CacheDirectory = "restic-destination-${destination.name}"; + CacheDirectoryMode = "0700"; + PrivateTmp = true; + ExecStartPre = pkgs.writeShellScript "restic-backup-${source.name}-to-${destination.name}-pre" '' + ${lib.optionalString (source.backupPre != null) source.backupPre} + ${lib.optionalString (source.paths != [ ]) '' + cat ${pkgs.writeText "staticPaths" (lib.concatLines source.paths)} >> ${filesFromTmpFile} + ''} + ${lib.optionalString (source.dynamicFilesFrom != null) '' + ${pkgs.writeScript "dynamicFilesFromScript" source.dynamicFilesFrom} >> ${filesFromTmpFile} + ''} + ''; + ExecStopPost = pkgs.writeShellScript "restic-backup-${source.name}-to-${destination.name}-post" '' + ${lib.optionalString (source.backupPost != null) source.backupPost} + ${lib.optionalString fileBackup '' + rm -f ${filesFromTmpFile} + ''} + ''; + Slice = "restic-destination-${destination.name}.slice"; + }; + }; + + destinationServices = + config.nixchad.resticModule.destinations + |> lib.mapAttrs' ( + name: value: { + name = "restic-destination-${name}"; + value = destinationTemplater name value; + } + ); + + transformMapping = + mapping: + # Resolves source/destination names to the config attrsets + { + # ["x"..] -> [{ name = "x"; unitOptions = ..; .. } ..] + sources = + mapping.sources |> lib.map (name: { inherit name; } // config.nixchad.resticModule.sources.${name}); + destinations = + mapping.destinations + |> lib.map (name: { inherit name; } // config.nixchad.resticModule.destinations.${name}); + } + # { sources = [x y]; destinations = [z w]; } -> [{ sources = x; destinations = z; } { sources = y; destinations = z; } ..] + |> lib.cartesianProduct + # Use singular attribute names from this point on + # [{ sources = x; destinations = z; } ..] -> [{ source = x; destination = z; } ..] + |> lib.map (attrset: { + source = attrset.sources; + destination = attrset.destinations; + }) + # [{ source = x; destination = z; } ..] -> [{ name = "restic-backup-source-to-destination"; value = { source = x; destination = z; }; } ..] + |> lib.map (attrset: { + name = "restic-backup-${attrset.source.name}-to-${attrset.destination.name}"; + value = backupTemplater attrset; + }) + # Turns that thing above into { "restic-backup-source-to-destination" = { source = x; .. }; ..} + |> lib.listToAttrs; + + backupServices = + config.nixchad.resticModule.mappings + # { mapping-name = { sources = [x y]; ..}; ..} -> { mapping-name = { "restic-backup-source-to-destination" = { source = x; ..}; ..}; ..} + |> lib.mapAttrs (_name: mapping: transformMapping mapping) + # Flatten attrs: { mapping-name = { "restic-backup-source-to-destination" = {..}; }; ..} -> { "restic-backup-source-to-destination" = {..}; ..} + |> lib.foldlAttrs ( + acc: _name: value: + acc // value + ) { }; + in + lib.mkIf config.nixchad.resticModule.enable { + assertions = lib.mapAttrsToList (name: source: { + assertion = lib.xor (source.paths != [ ] || source.dynamicFilesFrom != null) ( + source.command != [ ] + ); + message = "config.nixchad.resticModule.sources.${name} must specify exactly one of 'paths'/'dynamicFilesFrom' or 'command', but not both"; + }) config.nixchad.resticModule.sources; + + systemd.services = destinationServices // backupServices; + systemd.slices = + config.nixchad.resticModule.destinations + |> lib.mapAttrs (_: _: { sliceConfig.ConcurrencySoftMax = "1"; }); + }; +} diff --git a/nixosModules/common-services/restic/values.nix b/nixosModules/common-services/restic/values.nix new file mode 100644 index 00000000..5a08037e --- /dev/null +++ b/nixosModules/common-services/restic/values.nix @@ -0,0 +1,104 @@ +{ + config, + lib, + pkgs, + username, + ... +}: +let + photos = config.networking.hostName != "cloudberry"; + postgres = config.services.postgresql.enable; + vaultwarden = config.services.vaultwarden.enable; + paperless = config.services.paperless.enable; +in +{ + nixchad.resticModule = lib.mkIf config.nixchad.resticModule.enable { + sources = lib.mkMerge [ + { + secrets = { + paths = [ + "/secrets" + ]; + }; + stuff = { + paths = [ + "/home/${username}/Sync" + ]; + }; + } + (lib.mkIf photos { + photos = { + paths = [ + "/home/${username}/Pictures/Photos" + "/home/${username}/Pictures/Photos-phone" + ]; + }; + }) + (lib.mkIf postgres { + postgres = { + paths = [ + "/tmp/postgres" + ]; + backupPre = '' + echo 'creating temporary directory' + mkdir -p /tmp/postgres + echo 'dumping PostgreSQL database' + ${pkgs.shadow.su}/bin/su postgres -c ${config.services.postgresql.package}/bin/pg_dumpall > /tmp/postgres/pgdump.sql + ''; + backupPost = '' + rm -rfv /tmp/postgres + ''; + }; + }) + (lib.mkIf vaultwarden { + vaultwarden = { + paths = [ + "/tmp/vaultwarden" + ]; + backupPre = '' + echo 'creating temporary directory' + mkdir -p /tmp/vaultwarden + echo 'copying Vaultwarden data' + cp --reflink=auto -r /var/lib/bitwarden_rs /tmp/vaultwarden + ''; + backupPost = '' + rm -rfv /tmp/vaultwarden + ''; + }; + }) + (lib.mkIf paperless { + paperless = { + paths = [ + config.services.paperless.dataDir + ]; + }; + }) + ]; + + destinations = { + linus = { + settings = { + RESTIC_REPOSITORY = "sftp:kfears@sol.sphalerite.tech:/backup"; + RESTIC_PASSWORD_FILE = "/secrets/restic-backup-linus"; + }; + extraOptions = [ + "sftp.command='ssh kfears@sol.sphalerite.tech -i /home/${username}/.ssh/id_ed25519 -o StrictHostKeyChecking=no -s sftp'" + ]; + }; + }; + + mappings = { + linus = { + sources = [ + "secrets" + "stuff" + ] + ++ (lib.optional photos "photos") + ++ (lib.optional postgres "postgres") + ++ (lib.optional vaultwarden "vaultwarden") + ++ (lib.optional paperless "paperless"); + destinations = [ "linus" ]; + }; + }; + }; +} diff --git a/statix.toml b/statix.toml new file mode 100644 index 00000000..99b705f8 --- /dev/null +++ b/statix.toml @@ -0,0 +1,3 @@ +disabled = [ + "eta_reduction" +] diff --git a/suites/common-services.nix b/suites/common-services.nix index 3816dabf..a02829cb 100644 --- a/suites/common-services.nix +++ b/suites/common-services.nix @@ -3,6 +3,6 @@ nixchad = { smartctl-exporter.enable = lib.mkDefault true; alloy.enable = true; - restic.enable = true; + resticModule.enable = true; }; }