add support for replacing variable items value with environment variables

README.md

@@ -26,12 +26,13 @@ Usage:
   nomoperator bootstrap fs [path] [flags]
 
 Flags:
-      --base-dir string   Path to the base directory (default "./")
-      --delete            Enable delete missing jobs
-  -h, --help              help for fs
-      --path string       glob pattern relative to the base-dir (default "**/*.nomad")
-      --var-path string   var glob pattern relative to the base-dir (default "**/*.vars.yml")
-      --watch             Enable watch mode
+      --base-dir string       Path to the base directory (default "./")
+      --delete                Enable delete missing jobs
+      --env-template string   template used for environment variable replacement (default "<<\\$env\\(([A-Z0-9_]+)\\)>>")
+  -h, --help                  help for fs
+      --path string           glob pattern relative to the base-dir (default "**/*.nomad")
+      --var-path string       var glob pattern relative to the base-dir (default "**/*.vars.yml")
+      --watch                 Enable watch mode
 
 Global Flags:
   -a, --address string   Address of the Nomad server
@@ -55,6 +56,7 @@ Usage:
 Flags:
       --branch string                  git branch (default "main")
       --delete                         Enable delete missing jobs (default true)
+      --env-template string            template used for environment variable replacement (default "<<\\$env\\(([A-Z0-9_]+)\\)>>")
   -h, --help                           help for git
       --password string                SSH private key password
       --path string                    glob pattern relative to the repository root (default "**/*.nomad")
@@ -158,11 +160,26 @@ EOF
 
 ## Variables
 
-Variables are yml files. All keys and values in items should be of type string.
+Variables are yaml files. All keys and values in items should be of type string.
 
 ```yaml
 path: nomad/jobs/jobname
 items:
   key1: "value1"
   key2: "value2"
 ```
+
+To replace a value for items in the variable files by an environment variable, use the prefix defined in `--env-template` which defaults to `<<$env(VARIABLE)>>`.
+
+```yaml
+path: nomad/jobs/jobname
+items:
+  username: "john"
+  password: "<<$env(PASSWORD)>>"
+```
+
+You can use tools such as [dotenvx](https://dotenvx.com) to store encrypted environment variables. Then run nomoperator with dotenvx. Refer to the dotenvx documentation for more information.
+
+```bash
+dotenvx run -- nomoperator bootstrap fs --base-dir /path/to/base/dir --path jobs/*.nomad
+```

cmd/bootstrap_fs.go

@@ -9,11 +9,12 @@ import (
 )
 
 type fsFlags struct {
-	base_dir string
-	path     string
-	var_path string
-	watch    bool
-	delete   bool
+	base_dir     string
+	path         string
+	var_path     string
+	env_template string
+	watch        bool
+	delete       bool
 }
 
 var fsArgs fsFlags
@@ -23,6 +24,7 @@ func init() {
 	bootstrapFsCmd.Flags().StringVar(&fsArgs.base_dir, "base-dir", "./", "Path to the base directory")
 	bootstrapFsCmd.Flags().StringVar(&fsArgs.path, "path", "**/*.nomad", "glob pattern relative to the base-dir")
 	bootstrapFsCmd.Flags().StringVar(&fsArgs.var_path, "var-path", "**/*.vars.yml", "var glob pattern relative to the base-dir")
+	bootstrapFsCmd.Flags().StringVar(&fsArgs.env_template, "env-template", "<<\\$env\\(([A-Z0-9_]+)\\)>>", "template used for environment variable replacement")
 	bootstrapFsCmd.Flags().BoolVar(&fsArgs.watch, "watch", false, "Enable watch mode")
 	bootstrapFsCmd.Flags().BoolVar(&fsArgs.delete, "delete", false, "Enable delete missing jobs")
 }
@@ -33,10 +35,11 @@ var bootstrapFsCmd = &cobra.Command{
 	Long:  ``,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return reconcile.Run(reconcile.ReconcileOptions{
-			Path:    fsArgs.path,
-			VarPath: fsArgs.var_path,
-			Watch:   fsArgs.watch,
-			Delete:  fsArgs.delete,
+			Path:        fsArgs.path,
+			VarPath:     fsArgs.var_path,
+			EnvTemplate: fsArgs.env_template,
+			Watch:       fsArgs.watch,
+			Delete:      fsArgs.delete,
 			Fs: func() (billy.Filesystem, error) {
 				fs := osfs.New(fsArgs.base_dir)
 				return fs, nil

cmd/bootstrap_git.go

@@ -19,6 +19,7 @@ type gitFlags struct {
 	branch      string
 	path        string
 	var_path    string
+	env_prefix  string
 	username    string
 	password    string
 	sshkey      string
@@ -35,6 +36,7 @@ func init() {
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.branch, "branch", "main", "git branch")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.path, "path", "**/*.nomad", "glob pattern relative to the repository root")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.var_path, "var-path", "**/*.vars.yml", "var glob pattern relative to the repository root")
+	bootstrapGitCmd.Flags().StringVar(&fsArgs.env_template, "env-template", "<<\\$env\\(([A-Z0-9_]+)\\)>>", "template used for environment variable replacement")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.username, "username", "git", "SSH username")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.password, "password", "", "SSH private key password")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.sshkey, "ssh-key", "", "SSH private key")
@@ -50,10 +52,11 @@ var bootstrapGitCmd = &cobra.Command{
 	// Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return reconcile.Run(reconcile.ReconcileOptions{
-			Path:    gitArgs.path,
-			VarPath: gitArgs.var_path,
-			Watch:   gitArgs.watch,
-			Delete:  gitArgs.delete,
+			Path:        gitArgs.path,
+			VarPath:     gitArgs.var_path,
+			EnvTemplate: fsArgs.env_template,
+			Watch:       gitArgs.watch,
+			Delete:      gitArgs.delete,
 			Fs: func() (billy.Filesystem, error) {
 				repositoryURL, err := url.Parse(gitArgs.url)
 				if err != nil {

pkg/reconcile/reconcile.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
+	"regexp"
 	"strings"
 	"time"
 
@@ -18,11 +20,12 @@ import (
 )
 
 type ReconcileOptions struct {
-	Path    string
-	VarPath string
-	Watch   bool
-	Delete  bool
-	Fs      func() (billy.Filesystem, error)
+	Path        string
+	VarPath     string
+	EnvTemplate string
+	Watch       bool
+	Delete      bool
+	Fs          func() (billy.Filesystem, error)
 }
 
 func Run(opts ReconcileOptions) error {
@@ -90,6 +93,19 @@ func Run(opts ReconcileOptions) error {
 
 			// Update variable
 			fmt.Printf("Updating vars [%s]\n", newVariable.Path)
+
+			if opts.EnvTemplate != "" {
+				// Update variable items with environment variables
+				pattern := regexp.MustCompile(opts.EnvTemplate)
+				for key, value := range newVariable.Items {
+					replaced, newValue := replaceTemplateVariables(value, pattern)
+					if replaced {
+						fmt.Printf("Updating vars [%s] [%s] with environment variable\n", newVariable.Path, key)
+						newVariable.Items[key] = newValue
+					}
+				}
+			}
+
 			err = client.UpdateVariable(&newVariable)
 			if err != nil {
 				return err
@@ -250,3 +266,21 @@ func keys[K comparable, V any](m map[K]V) []K {
 	}
 	return keys
 }
+
+func replaceTemplateVariables(input string, pattern *regexp.Regexp) (bool, string) {
+	replaced := false
+
+	// Replace variables using the custom syntax
+	result := pattern.ReplaceAllStringFunc(input, func(match string) string {
+		replaced = true
+		// Extract variable name
+		name := pattern.FindStringSubmatch(match)[1]
+		// Get the environment variable value or use a default if not set
+		if value, exists := os.LookupEnv(name); exists {
+			return value
+		}
+		return "" // Default value if the variable is not found
+	})
+
+	return replaced, result
+}