nixcache: configure auth with devbox cache configure (#1961)

Add a hidden `devbox cache configure` command that sets up Nix so it can
authenticate with a private Devbox cache. This is a first step.
Ultimately, we'd like to do this configuration automatically before
building. We just need to make sure we don't re-prompt the user for sudo
or slow down commands like `devbox run` or `devbox shell`.

Configuring auth goes as follows:

1. Check to see if the Nix daemon is running. If it isn't, we don't need
to do anything.
2. Otherwise, we check to see if `~root/.aws/config` exists. If it does,
we assume that we've already configured the root user's AWS credentials.
This won't handle situations where the user already has a root AWS
config and can be improved.
3. If the AWS config doesn't exist, prompt the user to escalate to root.
If they say yes, relaunch `devbox cache configure`.
4. The new devbox process running as root writes an AWS config file that
tells the AWS CLI/SDKs to authenticate by running `devbox cache
credentials`.

Note that we need to use the default AWS profile. There's a bug in Nix
where non-default profiles (specified in the S3 URI with
?profile=devbox) don't use the default credential chain. If this gets
fixed, we should use a separate profile so that we don't interfere with
any AWS options set by the user (although hopefully this is rare for
root).

Author: gcurtis

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.27.7
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.7
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9
-	github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.23.4
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.4
 	github.com/bmatcuk/doublestar/v4 v4.6.1
@@ -40,7 +39,7 @@ require (
 	github.com/wk8/go-ordered-map/v2 v2.1.8
 	github.com/zealic/go2node v0.1.0
 	go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00
-	go.jetpack.io/pkg v0.0.0-20240404001923-7b42192bf9a5
+	go.jetpack.io/pkg v0.0.0-20240405214046-034a6476e201
 	go.jetpack.io/typeid v1.0.0
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/mod v0.16.0

go.sum

@@ -62,8 +62,6 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 h1:SIkD6T4zGQ+1YIit22wi37CGNkrE7mXV1vNA5VpI3TI=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4/go.mod h1:XfeqbsG0HNedNs0GT+ju4Bs+pFAwsrlzcRdMvdNVf5s=
-github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.23.4 h1:KuN2GQBLzac3PdhsVBt7n11jKfRsXg0OZSuuizF+yNw=
-github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.23.4/go.mod h1:OnFArLhSkVvZjmlx3wiYir/O44gpEerCXPJbK+LQBSE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 h1:NkHCgg0Ck86c5PTOzBZ0JRccI51suJDg5lgFtxBu1ek=
@@ -365,8 +363,8 @@ github.com/zealic/go2node v0.1.0 h1:ofxpve08cmLJBwFdI0lPCk9jfwGWOSD+s6216x0oAaA=
 github.com/zealic/go2node v0.1.0/go.mod h1:GrkFr+HctXwP7vzcU9RsgtAeJjTQ6Ud0IPCQAqpTfBg=
 go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00 h1:Kb+OlWOntAq+1nF+01ntqnQEqSJkFmLLS0RX5sl5zak=
 go.jetpack.io/envsec v0.0.16-0.20240329013200-4174c0acdb00/go.mod h1:dVG2n8fBAGpQczW8yk/6wuXb9uEhzaJF7wGXkGLRRCU=
-go.jetpack.io/pkg v0.0.0-20240404001923-7b42192bf9a5 h1:uFFlceGNlxqrKA/1umrBvNLTTBNkBU306Uqq1O27agM=
-go.jetpack.io/pkg v0.0.0-20240404001923-7b42192bf9a5/go.mod h1:gtmpVShXMEcZPBFZHswB3oCPYXobeR41b9CMybAjQYw=
+go.jetpack.io/pkg v0.0.0-20240405214046-034a6476e201 h1:59icpq6Y6uqnyG+IVijjGvLnL4U3syjFkq2ILnsC30Q=
+go.jetpack.io/pkg v0.0.0-20240405214046-034a6476e201/go.mod h1:gtmpVShXMEcZPBFZHswB3oCPYXobeR41b9CMybAjQYw=
 go.jetpack.io/typeid v1.0.0 h1:8gQ+iYGdyiQ0Pr40ydSB/PzMOIwlXX5DTojp1CBeSPQ=
 go.jetpack.io/typeid v1.0.0/go.mod h1:+UPEaECUgFxgAjFPn5Yf9eO/3ft/3xZ98Eahv9JW/GQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

internal/boxcli/cache.go

@@ -5,6 +5,7 @@ package boxcli
 
 import (
 	"encoding/json"
+	"os/user"
 
 	"github.com/MakeNowJust/heredoc/v2"
 	"github.com/pkg/errors"
@@ -33,9 +34,9 @@ func cacheCmd() *cobra.Command {
 		Short:   "upload specified or nix packages in current project to cache",
 		Long: heredoc.Doc(`
 			Upload specified nix installable or nix packages in current project to cache.
-			If [installable] is provided, only that installable will be uploaded. 
+			If [installable] is provided, only that installable will be uploaded.
 			Otherwise, all packages in the project will be uploaded.
-			To upload to specific cache, use --to flag. Otherwise, a cache from 
+			To upload to specific cache, use --to flag. Otherwise, a cache from
 			the cache provider will be used, if available.
 		`),
 		Args: cobra.MaximumNArgs(1),
@@ -61,41 +62,49 @@ func cacheCmd() *cobra.Command {
 		&flags.to, "to", "", "URI of the cache to copy to")
 
 	cacheCommand.AddCommand(uploadCommand)
+	cacheCommand.AddCommand(cacheConfigureCmd())
 	cacheCommand.AddCommand(cacheCredentialsCmd())
 	cacheCommand.Hidden = true
 
 	return cacheCommand
 }
 
+func cacheConfigureCmd() *cobra.Command {
+	username := ""
+	cmd := &cobra.Command{
+		Use:    "configure",
+		Short:  "Configure Nix to use the Devbox cache as a substituter",
+		Hidden: true,
+		Args:   cobra.MaximumNArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if username == "" {
+				u, _ := user.Current()
+				username = u.Username
+			}
+			return nixcache.Get().ConfigureAWS(cmd.Context(), username)
+		},
+	}
+	cmd.Flags().StringVar(&username, "user", "", "")
+	return cmd
+}
+
 func cacheCredentialsCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:    "credentials",
 		Short:  "Output S3 cache credentials",
 		Hidden: true,
 		Args:   cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg, err := nixcache.Get().Config(cmd.Context())
+			creds, err := nixcache.Get().Credentials(cmd.Context())
 			if err != nil {
 				return err
 			}
-
-			creds := struct {
-				Version         int    `json:"Version"`
-				AccessKeyID     string `json:"AccessKeyId"`
-				SecretAccessKey string `json:"SecretAccessKey"`
-				SessionToken    string `json:"SessionToken"`
-			}{
-				Version:         1,
-				AccessKeyID:     *cfg.Credentials.AccessKeyId,
-				SecretAccessKey: *cfg.Credentials.SecretKey,
-				SessionToken:    *cfg.Credentials.SessionToken,
-			}
 			out, err := json.Marshal(creds)
 			if err != nil {
 				return err
 			}
-			_, _ = cmd.OutOrStdout().Write(out)
-			return nil
+			_, err = cmd.OutOrStdout().Write(out)
+			return err
 		},
 	}
 }

internal/devbox/cache.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 
+	"go.jetpack.io/devbox/internal/boxcli/usererr"
 	"go.jetpack.io/devbox/internal/devbox/providers/nixcache"
 	"go.jetpack.io/devbox/internal/nix"
 )
@@ -12,38 +13,47 @@ func (d *Devbox) UploadProjectToCache(
 	ctx context.Context,
 	cacheURI string,
 ) error {
-	var err error
-	cacheConfig := nixcache.NixCacheConfig{URI: cacheURI}
-	if cacheConfig.URI == "" {
-		cacheConfig, err = d.providers.NixCache.Config(ctx)
+	if cacheURI == "" {
+		var err error
+		cacheURI, err = d.providers.NixCache.URI(ctx)
 		if err != nil {
 			return err
 		}
+		if cacheURI == "" {
+			return usererr.New("Your account's organization doesn't have a Nix cache.")
+		}
+	}
+
+	creds, err := d.providers.NixCache.Credentials(ctx)
+	if err != nil {
+		return err
 	}
 	profilePath, err := d.profilePath()
 	if err != nil {
 		return err
 	}
-
-	return nix.CopyInstallableToCache(
-		ctx,
-		d.stderr, cacheConfig.URI, profilePath, cacheConfig.CredentialsEnvVars())
+	return nix.CopyInstallableToCache(ctx, d.stderr, cacheURI, profilePath, creds.Env())
 }
 
 func UploadInstallableToCache(
 	ctx context.Context,
 	stderr io.Writer,
 	cacheURI, installable string,
 ) error {
-	var err error
-	cacheConfig := nixcache.NixCacheConfig{URI: cacheURI}
-	if cacheConfig.URI == "" {
-		cacheConfig, err = nixcache.Get().Config(ctx)
+	if cacheURI == "" {
+		var err error
+		cacheURI, err = nixcache.Get().URI(ctx)
 		if err != nil {
 			return err
 		}
+		if cacheURI == "" {
+			return usererr.New("Your account's organization doesn't have a Nix cache.")
+		}
+	}
+
+	creds, err := nixcache.Get().Credentials(ctx)
+	if err != nil {
+		return err
 	}
-	return nix.CopyInstallableToCache(
-		ctx,
-		stderr, cacheConfig.URI, installable, cacheConfig.CredentialsEnvVars())
+	return nix.CopyInstallableToCache(ctx, stderr, cacheURI, installable, creds.Env())
 }

internal/devbox/packages.go

@@ -445,19 +445,19 @@ func (d *Devbox) installNixPackagesToStore(ctx context.Context, mode installMode
 			flags = append(flags, "--refresh")
 		}
 
-		nixCacheConfig, err := d.providers.NixCache.Config(ctx)
-		if err != nil {
-			return err
+		args := &nix.BuildArgs{
+			AllowInsecure: pkg.HasAllowInsecure(),
+			Flags:         flags,
+			Writer:        d.stderr,
 		}
-
-		for _, installable := range installables {
-			args := &nix.BuildArgs{
-				AllowInsecure:    pkg.HasAllowInsecure(),
-				Env:              nixCacheConfig.CredentialsEnvVars(),
-				ExtraSubstituter: nixCacheConfig.URI,
-				Flags:            flags,
-				Writer:           d.stderr,
+		args.ExtraSubstituter, err = d.providers.NixCache.URI(ctx)
+		if err == nil {
+			creds, err := d.providers.NixCache.Credentials(ctx)
+			if err == nil {
+				args.Env = creds.Env()
 			}
+		}
+		for _, installable := range installables {
 			err = nix.Build(ctx, args, installable)
 			if err != nil {
 				fmt.Fprintf(d.stderr, "%s: ", stepMsg)