From c5ff55ea5b439f8ac046bf340a699b7444f32f31 Mon Sep 17 00:00:00 2001 From: Jaap de Haan <261428+jdehaan@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:03:24 +0000 Subject: [PATCH] feat: template --- .golangci.yml | 1 + backup/cmd/check-coverage.go | 3 +- backup/cmd/config.go | 3 +- backup/cmd/job.go | 19 +- backup/cmd/root.go | 1 + backup/cmd/test/commands_test.go | 178 ++++++++++ backup/internal/config.go | 192 ++++++++++- backup/internal/test/config_test.go | 499 ++++++++++++++++++++++++++++ backup/internal/testutil/builder.go | 96 +++++- backup/internal/testutil/config.go | 14 + docs/configuration.md | 38 ++- docs/templating.md | 205 ++++++++++++ 12 files changed, 1213 insertions(+), 36 deletions(-) create mode 100644 docs/templating.md diff --git a/.golangci.yml b/.golangci.yml index 47a18a3..cee266f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,6 +20,7 @@ linters: - fmt - io - log + - maps - os - path/filepath - sort diff --git a/backup/cmd/check-coverage.go b/backup/cmd/check-coverage.go index c0d775d..4e440ce 100644 --- a/backup/cmd/check-coverage.go +++ b/backup/cmd/check-coverage.go @@ -16,8 +16,9 @@ func buildCheckCoverageCommand(fs afero.Fs) *cobra.Command { Short: "Check path coverage", RunE: func(cmd *cobra.Command, args []string) error { configPath, _ := cmd.Flags().GetString("config") + overrides := parseSetFlags(cmd) - cfg, err := internal.LoadResolvedConfig(configPath) + cfg, err := internal.LoadResolvedConfig(configPath, overrides) if err != nil { return fmt.Errorf("loading config: %w", err) } diff --git a/backup/cmd/config.go b/backup/cmd/config.go index 1f5bd64..8fb2a6f 100644 --- a/backup/cmd/config.go +++ b/backup/cmd/config.go @@ -52,8 +52,9 @@ func buildConfigCommand() *cobra.Command { func configRunE(verb configVerb) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { configPath, _ := cmd.Flags().GetString("config") + overrides := parseSetFlags(cmd) - cfg, err := internal.LoadResolvedConfig(configPath) + cfg, err := internal.LoadResolvedConfig(configPath, overrides) if err != nil { return fmt.Errorf("%s: %w", verb.errCtx, err) } diff --git a/backup/cmd/job.go b/backup/cmd/job.go index b3e8b88..47c4b31 100644 --- a/backup/cmd/job.go +++ b/backup/cmd/job.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "strings" "time" "github.com/spf13/afero" @@ -25,6 +26,21 @@ type jobCommandOptions struct { createLogger LoggerFactory } +// parseSetFlags parses --set flag values (key=value) into a map. +func parseSetFlags(cmd *cobra.Command) map[string]string { + setFlags, _ := cmd.Flags().GetStringArray("set") + overrides := make(map[string]string, len(setFlags)) + + for _, s := range setFlags { + key, value, ok := strings.Cut(s, "=") + if ok { + overrides[key] = value + } + } + + return overrides +} + func buildJobCommand(fs afero.Fs, opts jobCommandOptions) *cobra.Command { return &cobra.Command{ Use: opts.use, @@ -32,8 +48,9 @@ func buildJobCommand(fs afero.Fs, opts jobCommandOptions) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { configPath, _ := cmd.Flags().GetString("config") rsyncPath, _ := cmd.Flags().GetString("rsync-path") + overrides := parseSetFlags(cmd) - cfg, err := internal.LoadResolvedConfig(configPath) + cfg, err := internal.LoadResolvedConfig(configPath, overrides) if err != nil { return fmt.Errorf("loading config: %w", err) } diff --git a/backup/cmd/root.go b/backup/cmd/root.go index 1f5527c..fe1d77b 100644 --- a/backup/cmd/root.go +++ b/backup/cmd/root.go @@ -27,6 +27,7 @@ func BuildRootCommandWithDeps(fs afero.Fs, shell internal.Exec) *cobra.Command { rootCmd.PersistentFlags().String("config", "config.yaml", "Path to the configuration file") rootCmd.PersistentFlags().String("rsync-path", "/usr/bin/rsync", "Path to the rsync binary") + rootCmd.PersistentFlags().StringArray("set", nil, "Set a variable override (key=value), can be repeated") rootCmd.AddCommand( buildListCommand(shell), diff --git a/backup/cmd/test/commands_test.go b/backup/cmd/test/commands_test.go index aab5365..a59236b 100644 --- a/backup/cmd/test/commands_test.go +++ b/backup/cmd/test/commands_test.go @@ -359,3 +359,181 @@ func TestRun_LoggerOpenDuringApply(t *testing.T) { assert.Contains(t, summaryContent, "STATUS [docs]: SUCCESS", "logger must remain open during cfg.Apply — proves defer cleanup() is function-scoped") } + +// --- --set flag --- + +func TestConfigShow_WithSetFlag(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + Source("/home/${user}").Target("/backup/${user}"). + Variable("user", "default"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + stdout, err := executeCommand(t, "config", "show", "--config", cfgPath, "--set", "user=alice") + + require.NoError(t, err) + assert.Contains(t, stdout, "alice_docs") + assert.Contains(t, stdout, "/home/alice/docs/") + assert.Contains(t, stdout, "/backup/alice/docs/") + assert.NotContains(t, stdout, "${user}") +} + +func TestConfigValidate_WithSetFlag(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + stdout, err := executeCommand(t, "config", "validate", "--config", cfgPath, "--set", "user=bob") + + require.NoError(t, err) + assert.Contains(t, stdout, "Configuration is valid.") +} + +func TestList_WithSetFlag(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + shell := &stubExec{output: []byte("rsync version 3.2.7 protocol version 31\n")} + + fs := afero.NewMemMapFs() + stdout, err := executeCommandWithDeps(t, fs, shell, "list", "--config", cfgPath, "--set", "user=alice") + + require.NoError(t, err) + assert.Contains(t, stdout, "Job: alice_docs") +} + +// --- template: variables validation at command level --- + +func TestConfigValidate_TemplateVarsMissing(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + TemplateVar("user").TemplateVar("user_cap"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + _, err := executeCommand(t, "config", "validate", "--config", cfgPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "missing required template variables") +} + +func TestConfigValidate_TemplateVarsProvidedViaSet(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + stdout, err := executeCommand(t, "config", "validate", "--config", cfgPath, "--set", "user=alice") + + require.NoError(t, err) + assert.Contains(t, stdout, "Configuration is valid.") +} + +func TestConfigShow_TemplateVarsResolved(t *testing.T) { + cfgPath := testutil.WriteConfigFile(t, testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build()) + + stdout, err := executeCommand(t, "config", "show", "--config", cfgPath, "--set", "user=alice") + + require.NoError(t, err) + assert.Contains(t, stdout, "alice_docs") + assert.NotContains(t, stdout, "${user}") +} + +// --- include at command level --- + +func TestConfigShow_WithInclude(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + stdout, err := executeCommand(t, "config", "show", "--config", mainPath) + + require.NoError(t, err) + assert.Contains(t, stdout, "alice_docs") + assert.Contains(t, stdout, "/home/alice/docs/") +} + +func TestConfigValidate_WithInclude(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "bob"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + stdout, err := executeCommand(t, "config", "validate", "--config", mainPath) + + require.NoError(t, err) + assert.Contains(t, stdout, "Configuration is valid.") +} + +func TestConfigValidate_IncludeMissingVars(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user").TemplateVar("user_cap"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := executeCommand(t, "config", "validate", "--config", mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "missing required template variables") +} + +func TestList_WithInclude(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + AddInclude("template.yaml", map[string]string{"user": "bob"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + shell := &stubExec{output: []byte("rsync version 3.2.7 protocol version 31\n")} + + stdout, err := executeCommandWithDeps(t, afero.NewMemMapFs(), shell, "list", "--config", mainPath) + + require.NoError(t, err) + assert.Contains(t, stdout, "Job: alice_docs") + assert.Contains(t, stdout, "Job: bob_docs") +} diff --git a/backup/internal/config.go b/backup/internal/config.go index f157e42..6162acc 100644 --- a/backup/internal/config.go +++ b/backup/internal/config.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "maps" "os" "path/filepath" "slices" @@ -15,15 +16,30 @@ import ( // Static errors for wrapping.. var ( - ErrJobValidation = errors.New("job validation failed") - ErrInvalidPath = errors.New("invalid path") - ErrPathValidation = errors.New("path validation failed") - ErrOverlappingPath = errors.New("overlapping path detected") - ErrJobFailure = errors.New("one or more jobs failed") + ErrJobValidation = errors.New("job validation failed") + ErrInvalidPath = errors.New("invalid path") + ErrPathValidation = errors.New("path validation failed") + ErrOverlappingPath = errors.New("overlapping path detected") + ErrJobFailure = errors.New("one or more jobs failed") + ErrMissingTemplateVars = errors.New("missing required template variables") + ErrNestedIncludes = errors.New("nested includes are not supported") ) +// Template declares required variables for a template config file. +type Template struct { + Variables []string `yaml:"variables"` +} + +// Include references a template config to instantiate with specific variable values. +type Include struct { + Uses string `yaml:"uses"` + With map[string]string `yaml:"with"` +} + // Config represents the overall backup configuration. type Config struct { + Template *Template `yaml:"template,omitempty"` + Include []Include `yaml:"include,omitempty"` Sources []Path `yaml:"sources"` Targets []Path `yaml:"targets"` Variables map[string]string `yaml:"variables"` @@ -98,9 +114,57 @@ func resolveField(input string, variables map[string]string) (string, error) { return resolved, nil } +const maxResolvePasses = 10 + +// ResolveVariables resolves variable-to-variable references within the variables map. +// Variables can reference other variables (e.g., source_home: "/home/${user}"). +// Performs multiple passes until no further substitutions occur or maxResolvePasses is reached. +func ResolveVariables(variables map[string]string) map[string]string { + resolved := make(map[string]string, len(variables)) + maps.Copy(resolved, variables) + + for range maxResolvePasses { + changed := false + + for k, v := range resolved { + newV := SubstituteVariables(v, resolved) + if newV != v { + resolved[k] = newV + changed = true + } + } + + if !changed { + break + } + } + + return resolved +} + func ResolveConfig(cfg Config) (Config, error) { resolvedCfg := cfg + resolvedCfg.Variables = ResolveVariables(cfg.Variables) + + for idx, source := range resolvedCfg.Sources { + resolved, err := resolveField(source.Path, resolvedCfg.Variables) + if err != nil { + return Config{}, fmt.Errorf("resolving source path %q: %w", source.Path, err) + } + + resolvedCfg.Sources[idx].Path = resolved + } + + for idx, target := range resolvedCfg.Targets { + resolved, err := resolveField(target.Path, resolvedCfg.Variables) + if err != nil { + return Config{}, fmt.Errorf("resolving target path %q: %w", target.Path, err) + } + + resolvedCfg.Targets[idx].Path = resolved + } + for idx := range resolvedCfg.Jobs { job := &resolvedCfg.Jobs[idx] @@ -108,13 +172,13 @@ func ResolveConfig(cfg Config) (Config, error) { var err error - job.Source, err = resolveField(job.Source, cfg.Variables) + job.Source, err = resolveField(job.Source, resolvedCfg.Variables) errs = append(errs, err) - job.Target, err = resolveField(job.Target, cfg.Variables) + job.Target, err = resolveField(job.Target, resolvedCfg.Variables) errs = append(errs, err) - job.Name, err = resolveField(job.Name, cfg.Variables) + job.Name, err = resolveField(job.Name, resolvedCfg.Variables) errs = append(errs, err) joined := errors.Join(errs...) @@ -200,7 +264,97 @@ func validateJobPaths(jobs []Job, pathType string, getPath func(job Job) string) return nil } -func LoadResolvedConfig(configPath string) (Config, error) { +// ValidateTemplateVars checks that all variables declared in the template section have values. +func ValidateTemplateVars(cfg Config) error { + if cfg.Template == nil || len(cfg.Template.Variables) == 0 { + return nil + } + + var missing []string + + for _, v := range cfg.Template.Variables { + if _, ok := cfg.Variables[v]; !ok { + missing = append(missing, v) + } + } + + if len(missing) > 0 { + return fmt.Errorf("%w: %v", ErrMissingTemplateVars, missing) + } + + return nil +} + +func loadTemplateConfig(templatePath string) (Config, error) { + templateFile, err := os.Open(templatePath) + if err != nil { + return Config{}, fmt.Errorf("failed to open: %w", err) + } + defer templateFile.Close() + + cfg, err := LoadConfig(templateFile) + if err != nil { + return Config{}, fmt.Errorf("failed to parse: %w", err) + } + + if len(cfg.Include) > 0 { + return Config{}, ErrNestedIncludes + } + + return cfg, nil +} + +func expandIncludes(cfg *Config, configDir string) error { + for _, inc := range cfg.Include { + templatePath := inc.Uses + if !filepath.IsAbs(templatePath) { + templatePath = filepath.Join(configDir, templatePath) + } + + tmplCfg, err := loadTemplateConfig(templatePath) + if err != nil { + return fmt.Errorf("include %q: %w", inc.Uses, err) + } + + if tmplCfg.Variables == nil { + tmplCfg.Variables = make(map[string]string) + } + + maps.Copy(tmplCfg.Variables, inc.With) + + err = ValidateTemplateVars(tmplCfg) + if err != nil { + return fmt.Errorf("include %q: %w", inc.Uses, err) + } + + resolved, err := ResolveConfig(tmplCfg) + if err != nil { + return fmt.Errorf("include %q: resolving config: %w", inc.Uses, err) + } + + cfg.Sources = append(cfg.Sources, resolved.Sources...) + cfg.Targets = append(cfg.Targets, resolved.Targets...) + cfg.Jobs = append(cfg.Jobs, resolved.Jobs...) + } + + cfg.Include = nil + + return nil +} + +func mergeOverrides(cfg Config, overrides []map[string]string) Config { + for _, override := range overrides { + if cfg.Variables == nil { + cfg.Variables = make(map[string]string) + } + + maps.Copy(cfg.Variables, override) + } + + return cfg +} + +func LoadResolvedConfig(configPath string, overrides ...map[string]string) (Config, error) { configFile, err := os.Open(configPath) if err != nil { return Config{}, fmt.Errorf("failed to open config: %w", err) @@ -212,9 +366,20 @@ func LoadResolvedConfig(configPath string) (Config, error) { return Config{}, fmt.Errorf("failed to parse YAML: %w", err) } - err = ValidateJobNames(cfg.Jobs) + cfg = mergeOverrides(cfg, overrides) + + return resolveAndValidate(cfg, filepath.Dir(configPath)) +} + +func resolveAndValidate(cfg Config, configDir string) (Config, error) { + err := expandIncludes(&cfg, configDir) + if err != nil { + return Config{}, fmt.Errorf("expanding includes: %w", err) + } + + err = ValidateTemplateVars(cfg) if err != nil { - return Config{}, fmt.Errorf("job validation failed: %w", err) + return Config{}, fmt.Errorf("template validation failed: %w", err) } resolvedCfg, err := ResolveConfig(cfg) @@ -222,6 +387,11 @@ func LoadResolvedConfig(configPath string) (Config, error) { return Config{}, fmt.Errorf("config resolution failed: %w", err) } + err = ValidateJobNames(resolvedCfg.Jobs) + if err != nil { + return Config{}, fmt.Errorf("job validation failed: %w", err) + } + err = ValidatePaths(resolvedCfg) if err != nil { return Config{}, fmt.Errorf("path validation failed: %w", err) diff --git a/backup/internal/test/config_test.go b/backup/internal/test/config_test.go index 2cd43ca..f914841 100644 --- a/backup/internal/test/config_test.go +++ b/backup/internal/test/config_test.go @@ -417,3 +417,502 @@ func TestConfigApply_VersionInfoError(t *testing.T) { assert.Contains(t, logBuf.String(), "Failed to fetch rsync version: command not found") assert.NotContains(t, logBuf.String(), "Rsync Binary Path") } + +func TestResolveVariables(t *testing.T) { + tests := []struct { + name string + input map[string]string + expected map[string]string + }{ + { + name: "NoReferences", + input: map[string]string{"a": "hello", "b": "world"}, + expected: map[string]string{"a": "hello", "b": "world"}, + }, + { + name: "SingleLevel", + input: map[string]string{"user": "alice", "home": "/home/${user}"}, + expected: map[string]string{"user": "alice", "home": "/home/alice"}, + }, + { + name: "MultiLevel", + input: map[string]string{ + "user": "bob", + "home": "/home/${user}", + "docs": "${home}/Documents", + "archive": "${docs}/archive", + }, + expected: map[string]string{ + "user": "bob", + "home": "/home/bob", + "docs": "/home/bob/Documents", + "archive": "/home/bob/Documents/archive", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := ResolveVariables(test.input) + assert.Equal(t, test.expected, result) + }) + } +} + +func TestResolveVariables_CircularReference(t *testing.T) { + result := ResolveVariables(map[string]string{"a": "${b}", "b": "${a}"}) + + // Circular references leave unresolved placeholders — exact values + // depend on map iteration order, so just verify they remain unresolved. + assert.Contains(t, result["a"], "${") + assert.Contains(t, result["b"], "${") +} + +func TestResolveConfig_ResolvesAllFields(t *testing.T) { + cfg := Config{ + Variables: map[string]string{ + "user": "alice", + "user_cap": "Jaap", + }, + Sources: []Path{{Path: "/home/${user}/"}}, + Targets: []Path{{Path: "/mnt/backup1/${user}"}}, + Jobs: []Job{ + { + Name: "${user}_docs", + Source: "/home/${user}/Documents/", + Target: "/mnt/backup1/${user}/documents", + }, + }, + } + + resolved, err := ResolveConfig(cfg) + + require.NoError(t, err) + assert.Equal(t, "/home/alice/", resolved.Sources[0].Path) + assert.Equal(t, "/mnt/backup1/alice", resolved.Targets[0].Path) + assert.Equal(t, "alice_docs", resolved.Jobs[0].Name) + assert.Equal(t, "/home/alice/Documents/", resolved.Jobs[0].Source) + assert.Equal(t, "/mnt/backup1/alice/documents", resolved.Jobs[0].Target) +} + +func TestResolveConfig_VariableChaining(t *testing.T) { + cfg := Config{ + Variables: map[string]string{ + "user": "bob", + "source_home": "/home/${user}", + "target_base": "/mnt/backup1/${user}", + }, + Sources: []Path{{Path: "/home/${user}/"}}, + Targets: []Path{{Path: "/mnt/backup1/${user}"}}, + Jobs: []Job{ + { + Name: "${user}_mail", + Source: "${source_home}/.thunderbird/", + Target: "${target_base}/mail", + }, + }, + } + + resolved, err := ResolveConfig(cfg) + + require.NoError(t, err) + assert.Equal(t, "/home/bob/", resolved.Sources[0].Path) + assert.Equal(t, "/mnt/backup1/bob", resolved.Targets[0].Path) + assert.Equal(t, "bob_mail", resolved.Jobs[0].Name) + assert.Equal(t, "/home/bob/.thunderbird/", resolved.Jobs[0].Source) + assert.Equal(t, "/mnt/backup1/bob/mail", resolved.Jobs[0].Target) +} + +func TestResolveConfig_SourceMacroError(t *testing.T) { + cfg := Config{ + Sources: []Path{{Path: "/home/@{bogus:val}/"}}, + Jobs: []Job{{Name: "job1", Source: "/src/", Target: "/dst/"}}, + } + + _, err := ResolveConfig(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving source path") +} + +func TestResolveConfig_TargetMacroError(t *testing.T) { + cfg := Config{ + Targets: []Path{{Path: "/backup/@{bogus:val}/"}}, + Jobs: []Job{{Name: "job1", Source: "/src/", Target: "/dst/"}}, + } + + _, err := ResolveConfig(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving target path") +} + +func TestLoadResolvedConfig_WithOverrides(t *testing.T) { + config := testutil.NewConfigBuilder(). + Source("/home/${user}"). + Target("/mnt/backup1/${user}"). + Variable("user", "default"). + AddJob("${user}_docs", "/home/${user}/docs/", "/mnt/backup1/${user}/docs/"). + Build() + + path := testutil.WriteConfigFile(t, config) + + cfg, err := LoadResolvedConfig(path, map[string]string{"user": "alice"}) + + require.NoError(t, err) + assert.Equal(t, "alice_docs", cfg.Jobs[0].Name) + assert.Equal(t, "/home/alice/docs/", cfg.Jobs[0].Source) + assert.Equal(t, "/mnt/backup1/alice/docs/", cfg.Jobs[0].Target) +} + +func TestLoadResolvedConfig_OverridesNewVariable(t *testing.T) { + config := testutil.NewConfigBuilder(). + Source("/home/${user}"). + Target("/mnt/backup1/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/mnt/backup1/${user}/docs/"). + Build() + + path := testutil.WriteConfigFile(t, config) + + cfg, err := LoadResolvedConfig(path, map[string]string{"user": "bob"}) + + require.NoError(t, err) + assert.Equal(t, "bob_docs", cfg.Jobs[0].Name) + assert.Equal(t, "/home/bob/docs/", cfg.Jobs[0].Source) + assert.Equal(t, "/mnt/backup1/bob/docs/", cfg.Jobs[0].Target) +} + +// --- ValidateTemplateVars --- + +func TestValidateTemplateVars(t *testing.T) { + tests := []struct { + name string + cfg Config + wantErr string + }{ + { + name: "NoTemplate", + cfg: Config{}, + }, + { + name: "EmptyVariablesList", + cfg: Config{Template: &Template{Variables: []string{}}}, + }, + { + name: "AllVariablesProvided", + cfg: Config{ + Template: &Template{Variables: []string{"user", "user_cap"}}, + Variables: map[string]string{"user": "alice", "user_cap": "Alice"}, + }, + }, + { + name: "OneMissing", + cfg: Config{ + Template: &Template{Variables: []string{"user", "user_cap"}}, + Variables: map[string]string{"user": "alice"}, + }, + wantErr: "missing required template variables: [user_cap]", + }, + { + name: "AllMissing", + cfg: Config{ + Template: &Template{Variables: []string{"user", "user_cap"}}, + Variables: map[string]string{}, + }, + wantErr: "missing required template variables", + }, + { + name: "NilVariablesMap", + cfg: Config{ + Template: &Template{Variables: []string{"user"}}, + }, + wantErr: "missing required template variables: [user]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateTemplateVars(test.cfg) + + if test.wantErr != "" { + require.Error(t, err) + require.ErrorIs(t, err, ErrMissingTemplateVars) + assert.Contains(t, err.Error(), test.wantErr) + } else { + assert.NoError(t, err) + } + }) + } +} + +// --- LoadResolvedConfig with template: variables --- + +func TestLoadResolvedConfig_TemplateVarsMissing(t *testing.T) { + config := testutil.NewConfigBuilder(). + TemplateVar("user").TemplateVar("user_cap"). + Source("/home/${user}").Target("/backup/${user}"). + Variable("user", "alice"). + AddJob("docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + + path := testutil.WriteConfigFile(t, config) + + _, err := LoadResolvedConfig(path) + + require.Error(t, err) + assert.Contains(t, err.Error(), "template validation failed") + assert.Contains(t, err.Error(), "user_cap") +} + +func TestLoadResolvedConfig_TemplateVarsProvidedViaOverride(t *testing.T) { + config := testutil.NewConfigBuilder(). + TemplateVar("user").TemplateVar("user_cap"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + + path := testutil.WriteConfigFile(t, config) + + cfg, err := LoadResolvedConfig(path, map[string]string{"user": "alice", "user_cap": "Alice"}) + + require.NoError(t, err) + assert.Equal(t, "alice_docs", cfg.Jobs[0].Name) +} + +func TestLoadResolvedConfig_TemplateVarsAllInYAML(t *testing.T) { + config := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + Variable("user", "bob"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + + path := testutil.WriteConfigFile(t, config) + + cfg, err := LoadResolvedConfig(path) + + require.NoError(t, err) + assert.Equal(t, "bob_docs", cfg.Jobs[0].Name) +} + +// --- LoadResolvedConfig with include --- + +func TestLoadResolvedConfig_BasicInclude(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + cfg, err := LoadResolvedConfig(mainPath) + + require.NoError(t, err) + require.Len(t, cfg.Jobs, 1) + assert.Equal(t, "alice_docs", cfg.Jobs[0].Name) + assert.Equal(t, "/home/alice/docs/", cfg.Jobs[0].Source) + assert.Equal(t, "/backup/alice/docs/", cfg.Jobs[0].Target) +} + +func TestLoadResolvedConfig_MultipleIncludes(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + AddInclude("template.yaml", map[string]string{"user": "bob"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + cfg, err := LoadResolvedConfig(mainPath) + + require.NoError(t, err) + require.Len(t, cfg.Jobs, 2) + assert.Equal(t, "alice_docs", cfg.Jobs[0].Name) + assert.Equal(t, "bob_docs", cfg.Jobs[1].Name) +} + +func TestLoadResolvedConfig_IncludeMissingTemplateVars(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user").TemplateVar("user_cap"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + // Only provides "user" but template requires "user" and "user_cap" + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := LoadResolvedConfig(mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "expanding includes") + assert.Contains(t, err.Error(), "missing required template variables") + assert.Contains(t, err.Error(), "user_cap") +} + +func TestLoadResolvedConfig_IncludeFileNotFound(t *testing.T) { + dir := t.TempDir() + + main := testutil.NewConfigBuilder(). + AddInclude("nonexistent.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := LoadResolvedConfig(mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "expanding includes") + assert.Contains(t, err.Error(), "failed to open") +} + +func TestLoadResolvedConfig_NestedIncludesRejected(t *testing.T) { + dir := t.TempDir() + + // inner template itself has an include (nested) + inner := testutil.NewConfigBuilder(). + AddInclude("other.yaml", map[string]string{"x": "y"}). + Source("/src").Target("/dst"). + AddJob("inner", "/src/a/", "/dst/a/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "inner.yaml", inner) + + main := testutil.NewConfigBuilder(). + AddInclude("inner.yaml", map[string]string{}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := LoadResolvedConfig(mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "expanding includes") + assert.Contains(t, err.Error(), "nested includes are not supported") +} + +func TestLoadResolvedConfig_IncludeInvalidYAML(t *testing.T) { + dir := t.TempDir() + + testutil.WriteConfigFileInDir(t, dir, "bad.yaml", "{{{not valid yaml") + + main := testutil.NewConfigBuilder(). + AddInclude("bad.yaml", map[string]string{}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := LoadResolvedConfig(mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "expanding includes") + assert.Contains(t, err.Error(), "failed to parse") +} + +func TestLoadResolvedConfig_IncludeWithVariableChaining(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + Variable("home", "/home/${user}"). + AddJob("${user}_mail", "${home}/.thunderbird/", "/backup/${user}/mail"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + cfg, err := LoadResolvedConfig(mainPath) + + require.NoError(t, err) + require.Len(t, cfg.Jobs, 1) + assert.Equal(t, "alice_mail", cfg.Jobs[0].Name) + assert.Equal(t, "/home/alice/.thunderbird/", cfg.Jobs[0].Source) + assert.Equal(t, "/backup/alice/mail", cfg.Jobs[0].Target) +} + +func TestLoadResolvedConfig_IncludeWithOverridesOnMainConfig(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/${target_root}/${user}"). + Variable("target_root", "backup"). + AddJob("${user}_docs", "/home/${user}/docs/", "/${target_root}/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + cfg, err := LoadResolvedConfig(mainPath) + + require.NoError(t, err) + assert.Equal(t, "/backup/alice/docs/", cfg.Jobs[0].Target) +} + +func TestLoadResolvedConfig_IncludeMergesSources(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/${user}/docs/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + Source("/shared").Target("/shared-backup"). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + AddJob("shared_data", "/shared/data/", "/shared-backup/data/"). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + cfg, err := LoadResolvedConfig(mainPath) + + require.NoError(t, err) + require.Len(t, cfg.Jobs, 2) + + // Main config's own sources + included template's sources + assert.Len(t, cfg.Sources, 2) + assert.Len(t, cfg.Targets, 2) +} + +func TestLoadResolvedConfig_IncludeMacroError(t *testing.T) { + dir := t.TempDir() + + template := testutil.NewConfigBuilder(). + TemplateVar("user"). + Source("/home/${user}").Target("/backup/${user}"). + AddJob("${user}_docs", "/home/@{bogus:val}/", "/backup/${user}/docs/"). + Build() + testutil.WriteConfigFileInDir(t, dir, "template.yaml", template) + + main := testutil.NewConfigBuilder(). + AddInclude("template.yaml", map[string]string{"user": "alice"}). + Build() + mainPath := testutil.WriteConfigFileInDir(t, dir, "main.yaml", main) + + _, err := LoadResolvedConfig(mainPath) + + require.Error(t, err) + assert.Contains(t, err.Error(), "expanding includes") +} diff --git a/backup/internal/testutil/builder.go b/backup/internal/testutil/builder.go index a2cf083..655f51d 100644 --- a/backup/internal/testutil/builder.go +++ b/backup/internal/testutil/builder.go @@ -17,12 +17,19 @@ type jobDef struct { exclusions []string } +type includeDef struct { + uses string + with map[string]string +} + // ConfigBuilder constructs YAML config strings declaratively. type ConfigBuilder struct { - sources []string - targets []string - variables map[string]string - jobs []jobDef + sources []string + targets []string + variables map[string]string + jobs []jobDef + templateVars []string + includes []includeDef } // NewConfigBuilder creates an empty ConfigBuilder. @@ -65,10 +72,27 @@ func (b *ConfigBuilder) AddJob(name, source, target string, opts ...JobOpt) *Con return b } +// TemplateVar adds a required template variable name. +func (b *ConfigBuilder) TemplateVar(name string) *ConfigBuilder { + b.templateVars = append(b.templateVars, name) + + return b +} + +// AddInclude adds a template include with variable bindings. +func (b *ConfigBuilder) AddInclude(uses string, with map[string]string) *ConfigBuilder { + b.includes = append(b.includes, includeDef{uses: uses, with: with}) + + return b +} + // Build produces the YAML config string. func (b *ConfigBuilder) Build() string { var result strings.Builder + b.writeTemplate(&result) + b.writeIncludes(&result) + result.WriteString("sources:\n") for _, s := range b.sources { @@ -92,28 +116,64 @@ func (b *ConfigBuilder) Build() string { result.WriteString("jobs:\n") for _, job := range b.jobs { - fmt.Fprintf(&result, " - name: %q\n", job.name) - fmt.Fprintf(&result, " source: %q\n", job.source) - fmt.Fprintf(&result, " target: %q\n", job.target) + writeJob(&result, job) + } - if job.delete != nil { - fmt.Fprintf(&result, " delete: %v\n", *job.delete) - } + return result.String() +} - if job.enabled != nil { - fmt.Fprintf(&result, " enabled: %v\n", *job.enabled) - } +func (b *ConfigBuilder) writeTemplate(writer *strings.Builder) { + if len(b.templateVars) == 0 { + return + } - if len(job.exclusions) > 0 { - result.WriteString(" exclusions:\n") + writer.WriteString("template:\n variables:\n") - for _, e := range job.exclusions { - fmt.Fprintf(&result, " - %q\n", e) + for _, v := range b.templateVars { + fmt.Fprintf(writer, " - %q\n", v) + } +} + +func (b *ConfigBuilder) writeIncludes(writer *strings.Builder) { + if len(b.includes) == 0 { + return + } + + writer.WriteString("include:\n") + + for _, inc := range b.includes { + fmt.Fprintf(writer, " - uses: %q\n", inc.uses) + + if len(inc.with) > 0 { + writer.WriteString(" with:\n") + + for k, v := range inc.with { + fmt.Fprintf(writer, " %s: %q\n", k, v) } } } +} - return result.String() +func writeJob(writer *strings.Builder, job jobDef) { + fmt.Fprintf(writer, " - name: %q\n", job.name) + fmt.Fprintf(writer, " source: %q\n", job.source) + fmt.Fprintf(writer, " target: %q\n", job.target) + + if job.delete != nil { + fmt.Fprintf(writer, " delete: %v\n", *job.delete) + } + + if job.enabled != nil { + fmt.Fprintf(writer, " enabled: %v\n", *job.enabled) + } + + if len(job.exclusions) > 0 { + writer.WriteString(" exclusions:\n") + + for _, e := range job.exclusions { + fmt.Fprintf(writer, " - %q\n", e) + } + } } // Enabled sets the enabled flag on a job. diff --git a/backup/internal/testutil/config.go b/backup/internal/testutil/config.go index 9b0c3ec..9d0a4f2 100644 --- a/backup/internal/testutil/config.go +++ b/backup/internal/testutil/config.go @@ -23,3 +23,17 @@ func WriteConfigFile(t *testing.T, content string) string { return path } + +// WriteConfigFileInDir writes YAML content to a named file in the given directory. +func WriteConfigFileInDir(t *testing.T, dir, name, content string) string { + t.Helper() + + path := filepath.Join(dir, name) + + err := os.WriteFile(path, []byte(content), internal.LogFilePermission) + if err != nil { + t.Fatalf("failed to write config file: %v", err) + } + + return path +} diff --git a/docs/configuration.md b/docs/configuration.md index c2bf0d4..ffaa9f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,10 +5,12 @@ The backup tool is configured using a YAML file, typically named `sync.yaml`. Th ## Top-Level Structure ```yaml -sources: # List of source paths to back up -targets: # List of target paths for backups -variables: # Key-value pairs for variable substitution -jobs: # List of backup jobs +template: # (Optional) Declares required variables for this template +include: # (Optional) List of template configs to instantiate +sources: # List of source paths to back up +targets: # List of target paths for backups +variables: # Key-value pairs for variable substitution +jobs: # List of backup jobs ``` ## Sources and Targets @@ -46,6 +48,33 @@ jobs: target: "/backup/@{capitalize:${user}}/docs" # resolves to /backup/Alice/docs ``` +## Template (Optional) + +Declares which variables a config file requires. When present, the tool validates +that every listed variable has a value before resolving. See +[templating.md](templating.md) for details. + +```yaml +template: + variables: + - user + - user_cap +``` + +## Include (Optional) + +Instantiate one or more template configs with specific variable bindings. Each +entry references a template file and provides the required variables. See +[templating.md](templating.md) for details. + +```yaml +include: + - uses: user_template.yaml + with: + user: alice + user_cap: Alice +``` + ## Jobs Each job defines a backup operation: @@ -101,3 +130,4 @@ jobs: - Exclusions are relative to the specified source or target path. - Jobs with `enabled: false` are ignored. - If `delete` is omitted, it defaults to `true` (target files not present in source will be deleted from the destination). +- For templating features (`template:`, `include:`, `--set` flags), see [templating.md](templating.md). diff --git a/docs/templating.md b/docs/templating.md new file mode 100644 index 0000000..7dd020a --- /dev/null +++ b/docs/templating.md @@ -0,0 +1,205 @@ +# Configuration Templating + +## Overview + +Configuration files support a variable substitution system that enables a single +template to serve multiple use cases. Variables defined in the YAML `variables` +section can be overridden from the command line using the `--set` flag, turning +any config file into a reusable template. + +A second mechanism, **includes**, allows a main config to instantiate a template +multiple times with different variable bindings — similar to GitHub Actions' +`uses`/`with` pattern. This replaces per-user (or per-host) config files with a +single template plus a lightweight orchestration file. + +## Variable Substitution + +Variables are referenced with `${variable_name}` syntax and can appear in: + +- **Job fields**: `name`, `source`, `target` +- **Source and target paths** (top-level `sources` / `targets` sections) +- **Other variables** (variable-to-variable references) + +### Variable Resolution Order + +1. Variables defined in the YAML `variables` section are loaded +2. CLI `--set` overrides are merged in (overwriting any matching keys) +3. Variable self-references are resolved (multi-pass, up to 10 iterations) +4. All config fields are substituted using the fully resolved variables + +This means variables can reference other variables: + +```yaml +variables: + source_home: "/home/${user}" + target_base: "/mnt/backup1/${user}" + +jobs: + - name: "${user}_documents" + source: "${source_home}/Documents/" + target: "${target_base}/documents" +``` + +When invoked with `--set user=alice`, the resolution chain is: + +1. `user` = `alice` (from CLI) +2. `source_home` = `/home/${user}` → `/home/alice` +3. `target_base` = `/mnt/backup1/${user}` → `/mnt/backup1/alice` +4. Job name = `${user}_documents` → `alice_documents` +5. Job source = `${source_home}/Documents/` → `/home/alice/Documents/` + +## Declaring Required Variables (`template:`) + +A config file can declare which variables it **requires** using a `template:` +section. When present, the tool validates that every listed variable has a value +before resolving the config — either from the YAML `variables:` section, a +`--set` flag, or an `include:` `with:` block. + +```yaml +template: + variables: + - user + - user_cap +``` + +If any declared variable is missing, the tool exits with an error listing the +unset variables. This makes template requirements explicit and catches typos or +forgotten flags early. + +## Includes (`include:`) + +A main config can instantiate one or more templates using the `include:` section. +Each entry specifies: + +- **`uses`**: path to the template config file (relative to the main config's + directory, or absolute) +- **`with`**: map of variable values to inject into the template + +```yaml +include: + - uses: user_template.yaml + with: + user: alice + user_cap: Alice + + - uses: user_template.yaml + with: + user: bob + user_cap: Bob +``` + +### How includes work + +1. Each `include` entry loads the referenced template file +2. The `with` values are merged into the template's `variables` map +3. Template variable validation runs (all `template.variables` must be set) +4. The template is resolved (variable substitution) +5. The resolved sources, targets, and jobs are appended to the main config +6. After all includes are expanded, the main config goes through standard + validation (job names, paths, overlaps) + +### Constraints + +- **No nested includes**: a template referenced via `include` cannot itself + contain `include` entries. This keeps the system simple and predictable. +- **Include paths are relative** to the directory containing the main config + file, unless an absolute path is specified. +- A main config can have both its own `sources`/`targets`/`jobs` and `include` + entries — they are merged together. + +### Example: multi-user orchestration + +**Template** (`user_template.yaml`): + +```yaml +template: + variables: + - user + - user_cap + +sources: + - path: "/home/${user}/" + - path: "/home/data/family/${user_cap}/" + +targets: + - path: "/mnt/backup1/${user}" + +variables: + source_home: "/home/${user}" + target_base: "/mnt/backup1/${user}" + +jobs: + - name: "${user}_mail" + source: "${source_home}/.thunderbird/" + target: "${target_base}/mail" + - name: "${user}_documents" + source: "${source_home}/Documents/" + target: "${target_base}/documents" +``` + +**Main config** (`users.yaml`): + +```yaml +include: + - uses: user_template.yaml + with: + user: alice + user_cap: Alice + + - uses: user_template.yaml + with: + user: bob + user_cap: Bob +``` + +Running `backup run --config users.yaml` expands both includes and executes all +jobs for both users in a single invocation. + +## CLI Usage + +The `--set` flag can be used with any command (`list`, `run`, `simulate`, +`config show`, `config validate`, `check-coverage`): + +```sh +# Show resolved config for user "alice" +backup config show --config user_template.yaml --set user=alice --set user_cap=Alice + +# Simulate backup for user "bob" +backup simulate --config user_template.yaml --set user=bob --set user_cap=Bob + +# Run backup for user "alice" +backup run --config user_template.yaml --set user=alice --set user_cap=Alice + +# Run all users via the orchestration config (no --set needed) +backup run --config users.yaml +``` + +Multiple `--set` flags can be specified. Later values override earlier ones for +the same key. + +## Choosing Between `--set` and `include:` + +| Approach | Best for | Example | +| -------------------- | ------------------------------------------- | -------------------------------------------- | +| `--set` flags | Ad-hoc CLI usage, CI scripts, single user | `backup run --config tpl.yaml --set user=bob` | +| `include:` in config | Multi-user/multi-host, declarative setups | `backup run --config users.yaml` | +| Combined | Includes with a shared override | `backup run --config users.yaml --set base=/mnt/nfs` | + +Both approaches can be combined: `--set` overrides apply to the main config's +variables before includes are expanded. Variables defined inside a template's +`with:` block are scoped to that template instance only. + +## Design Notes + +- **Backward compatible**: Configs without `${…}` placeholders work unchanged. + The `--set` flag and `template:`/`include:` sections are all optional. +- **Override semantics**: `--set` values take precedence over values defined in + the YAML `variables` section. +- **Multi-pass resolution**: Variable self-references are resolved iteratively + (up to 10 passes). Circular references are left unresolved rather than + causing an error. +- **Validation**: Template variable validation runs before resolution to catch + missing variables early. Job name validation (uniqueness, character checks) + and path validation run on fully resolved configs. +- **No nested includes**: Keeping the include depth to one level avoids + complexity and makes configs easy to reason about.