diff --git a/Jockerfile b/Jockerfile index 9a894ab..d800048 100644 --- a/Jockerfile +++ b/Jockerfile @@ -1,40 +1,40 @@ #syntax=ghcr.io/jocker-org/jocker -local std = import "std.libsonnet"; +// local std = import "std.libsonnet"; { "stages": [ { "name": "builder", - "from": "alpine:latest", + "from": "golang:alpine", "steps": [ - std.stage.step.workdir("/src"), - // {"type": "WORKDIR", "path": "/src"}, + // std.stage.step.workdir("/src"), + {"type": "WORKDIR", "path": "/src"}, {"type": "RUN", "command": "apk update"}, {"type": "RUN", "command": "apk upgrade"}, - {"type": "RUN", "command": "apk add mkdocs"}, - {"type": "COPY", "src": "mkdocs.yml", "dst": "/src"}, - {"type": "COPY", "src": "./docs", "dst": "/src/docs"}, - {"type": "RUN", "command": "mkdocs build"} + // {"type": "RUN", "command": "apk add mkdocs"}, + // {"type": "COPY", "src": "mkdocs.yml", "dst": "/src"}, + // {"type": "COPY", "src": "./docs", "dst": "/src/docs"}, + // {"type": "RUN", "command": "mkdocs build"} ] }, - { - "name": "server", - "from": "alpine:latest", - "steps": [ - {"type": "RUN", "command": "addgroup -g 1000 app && adduser -G app -u 1000 app -D"}, - {"type": "RUN", "command": "apk update"}, - {"type": "RUN", "command": "apk upgrade"}, - {"type": "RUN", "command": "apk add darkhttpd"}, - {"type": "COPY", "from": "builder", "src": "/src/site", "dst": "/www"}, - {"type": "USER", "user": "1000"}, - ], - }, + // { + // "name": "server", + // "from": "alpine:latest", + // "steps": [ + // {"type": "RUN", "command": "addgroup -g 1000 app && adduser -G app -u 1000 app -D"}, + // {"type": "RUN", "command": "apk update"}, + // {"type": "RUN", "command": "apk upgrade"}, + // {"type": "RUN", "command": "apk add darkhttpd"}, + // {"type": "COPY", "from": "builder", "src": "/src/site", "dst": "/www"}, + // {"type": "USER", "user": "1000"}, + // ], + // }, ], - "image": { - "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], - "Cmd": ["darkhttpd", "/www"], - }, + // "image": { + // "Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"], + // "Cmd": ["darkhttpd", "/www"], + // }, "excludes" : ["*", "!docs", "!mkdocs.yml"], } diff --git a/cmd/jocker/debug-dump.go b/cmd/jocker/debug-dump.go index 5a7a95b..b557f31 100644 --- a/cmd/jocker/debug-dump.go +++ b/cmd/jocker/debug-dump.go @@ -14,26 +14,22 @@ func DebugDump() error { vm := jsonnet.MakeVM() vm.Importer(&jsonnet.FileImporter{JPaths: []string{"/lib/"}}) - // Initialize Jsonnet VM and evaluate Jockerfile jsonStr, err := vm.EvaluateFile("Jockerfile") if err != nil { log.Fatal(err) } - // Parse JSON into Jockerfile struct j, err := parser.ParseJockerfile(jsonStr) if err != nil { log.Fatal(err) } - - // Generate LLB state from Jockerfile - state := j.ToLLB() ctx := context.TODO() + state := j.ToLLB("all", ctx, nil) + dt, err := state.Marshal(ctx, llb.LinuxAmd64) if err != nil { log.Fatal(err) } - // Write LLB definition to stdout return llb.WriteTo(dt, os.Stdout) } diff --git a/flake.nix b/flake.nix index 9b1e658..3951c28 100644 --- a/flake.nix +++ b/flake.nix @@ -3,36 +3,52 @@ inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.1.*.tar.gz"; - outputs = { self, nixpkgs }: + outputs = + { self, nixpkgs }: let goVersion = 22; # Change this to update the whole stack - supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; - forEachSupportedSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f { - pkgs = import nixpkgs { - inherit system; - overlays = [ self.overlays.default ]; - }; - }); + supportedSystems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + forEachSupportedSystem = + f: + nixpkgs.lib.genAttrs supportedSystems ( + system: + f { + pkgs = import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + }; + } + ); in { overlays.default = final: prev: { go = final."go_1_${toString goVersion}"; }; - devShells = forEachSupportedSystem ({ pkgs }: { - default = pkgs.mkShell { - packages = with pkgs; [ - # go (version is specified by overlay) - go + devShells = forEachSupportedSystem ( + { pkgs }: + { + default = pkgs.mkShell { + packages = with pkgs; [ + # go (version is specified by overlay) + go - # goimports, godoc, etc. - gotools gopls + buildkit + # goimports, godoc, etc. + gotools + gopls - # https://github.com/golangci/golangci-lint - golangci-lint - ]; - }; - }); + # https://github.com/golangci/golangci-lint + golangci-lint + ]; + }; + } + ); }; } diff --git a/internal/parser/builder.go b/internal/parser/builder.go index 3f52987..004f58d 100644 --- a/internal/parser/builder.go +++ b/internal/parser/builder.go @@ -14,14 +14,20 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/gateway/client" - specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) func readFile(ctx context.Context, c client.Client, filename string) (content []byte, err error) { src := llb.Local("context", - llb.IncludePatterns([]string{filename}), + llb.IncludePatterns([]string{ + filename, + "**/" + filename, + "*/", + }), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("Jockerfile"), + llb.WithCustomName("load " + filename), + ) def, err := src.Marshal(ctx) @@ -60,6 +66,11 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } } + jopts, err := json.Marshal(opts) + if err != nil { + return nil, fmt.Errorf("failed to marshal opts: %w", err) + } + jbuildargs, err := json.Marshal(buildargs) if err != nil { return nil, fmt.Errorf("failed to marshal buildargs: %w", err) @@ -67,8 +78,17 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { vm := jsonnet.MakeVM() vm.ExtCode("buildArgs", string(jbuildargs)) + vm.ExtCode("filename", fmt.Sprintf("%q",filename)) + vm.ExtCode("opts", fmt.Sprintf("%q",jopts)) vm.Importer(NewChainedImporter(NewContextImporter(ctx, c), []string{"/lib/"})) - jsonStr, err := vm.EvaluateFile(filename) + + dtJockerfile, err := readFile(ctx, c, filename) + if err != nil { + return nil, errors.Wrapf(err, "failed to read Jockerfile") + } + + // jsonStr, err := vm.EvaluateFile(filename) + jsonStr, err := vm.EvaluateAnonymousSnippet(filename, string(dtJockerfile)) if err != nil { return nil, err } @@ -84,7 +104,8 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { j.Excludes, _ = dockerignore.Parse(bytes.NewReader(content)) } } - state := j.ToLLB() + + state := j.ToLLB(buildargs["debug"], ctx, c) dt, err := state.Marshal(ctx, llb.LinuxAmd64) if err != nil { @@ -96,7 +117,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { }) if err != nil { - return nil, fmt.Errorf("failed to resolve dockerfile: %w", err) + return nil, fmt.Errorf("failed to resolve jockerfile: %w", err) } ref, err := res.SingleRef() @@ -111,17 +132,15 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { userplatform, err) } } - img := &specs.Image{ - Platform: p, - Config: j.Image, - } - config, err := json.Marshal(img) + j.Image.Platform = p + + config, err := json.Marshal(j.Image) if err != nil { - return nil, fmt.Errorf("failed to marshal image config: %w", err) + return nil, errors.Wrapf(err, "failed to marshal image config") } - res.AddMeta(exptypes.ExporterImageConfigKey, config) res.SetRef(ref) + res.AddMeta(exptypes.ExporterImageConfigKey, config) return res, nil } diff --git a/internal/parser/jockerfile.go b/internal/parser/jockerfile.go index 2d37293..c094cf0 100644 --- a/internal/parser/jockerfile.go +++ b/internal/parser/jockerfile.go @@ -2,8 +2,8 @@ package parser import ( "encoding/json" - - specs "github.com/opencontainers/image-spec/specs-go/v1" + dockerspec "github.com/moby/docker-image-spec/specs-go/v1" + // specs "github.com/opencontainers/image-spec/specs-go/v1" ) type ArgStep struct { @@ -37,7 +37,7 @@ type BuildStage struct { type Jockerfile struct { Stages []BuildStage `json:"stages"` - Image specs.ImageConfig `json:"image"` + Image dockerspec.DockerOCIImage `json:image` Excludes []string `json:"excludes,omitempty"` } diff --git a/internal/parser/llb.go b/internal/parser/llb.go index 7356eea..a590ff9 100644 --- a/internal/parser/llb.go +++ b/internal/parser/llb.go @@ -1,22 +1,77 @@ package parser import ( + "context" + "crypto/rand" + "encoding/json" "fmt" - "log" + "log/slog" + "strings" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/client/llb/imagemetaresolver" + "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/frontend/gateway/client" + + gw "github.com/moby/buildkit/frontend/gateway/client" + dockerspec "github.com/moby/docker-image-spec/specs-go/v1" ) type BuildContext struct { stages map[string]llb.State state llb.State context llb.State + debug string + ctx context.Context + image dockerspec.DockerOCIImage } type BuildStep interface { Evaluate(*BuildContext) llb.State } +func parseKeyValue(env string) (string, string) { + parts := strings.SplitN(env, "=", 2) + v := "" + if len(parts) > 1 { + v = parts[1] + } + + return parts[0], v +} + +// normalizeImage returns the image with the docker.io/library prefixed if it's not +// from a registry. Otherwise we can't resolve the Image Metadata +func normalizeImage(image string) string { + parts := strings.Split(image, "/") + + if len(parts) > 1 && (strings.Contains(parts[0], ".") || strings.Contains(parts[0], ":")) { + return image + } + + if len(parts) == 1 { + return "docker.io/library/" + image + } + + return "docker.io/" + image +} + +// debugLog conditionally logs a debug message and injects an echo command into the LLB state. +// it also adds a random uuid for invalidating docker cache +// https://github.com/docker/buildx/issues/2387 +func debugLog(b *BuildContext, msg string) { + by := make([]byte, 4) + _, err := rand.Read(by) + if err != nil { + slog.Error("error while creating unique id", "error", err) + } + uuid := fmt.Sprintf("%x ", by) + if b.debug == "all" { + slog.Warn("Step: ", "buildctx", msg) + b.state = b.state.Run(llb.Shlex("echo DEBUG " + uuid + msg)).Root() + } +} + func (c *ArgStep) Evaluate(b *BuildContext) llb.State { b.state = b.state.AddEnv(c.Name, c.Value) return b.state @@ -49,11 +104,14 @@ func (c *RunStep) Evaluate(b *BuildContext) llb.State { return b.state } + debugLog(b, c.Command) + b.state = b.state.Run(shf(c.Command)).Root() return b.state } func (c *WorkdirStep) Evaluate(b *BuildContext) llb.State { + debugLog(b, c.Path) b.state = b.state.Dir(c.Path) return b.state } @@ -67,24 +125,67 @@ func shf(cmd string, v ...interface{}) llb.RunOption { return llb.Args([]string{"/bin/sh", "-c", fmt.Sprintf(cmd, v...)}) } -func (stage *BuildStage) ToLLB(b *BuildContext) llb.State { +func (stage *BuildStage) ToLLB(b *BuildContext, c client.Client) llb.State { if stage.From == "scratch" { b.state = llb.Scratch() } else { + var img dockerspec.DockerOCIImage + baseImg := normalizeImage(stage.From) + if c == nil { + metaresolver := imagemetaresolver.Default() + _, _, dt, err := metaresolver.ResolveImageConfig(b.ctx, baseImg, sourceresolver.Opt{ + ImageOpt: &sourceresolver.ResolveImageOpt{ + ResolveMode: llb.ResolveModeDefault.String(), + }, + }) + if err != nil { + debugLog(b, "failed to resolve image") + slog.Error("failed to resolve image", "FROM", err) + } + if err := json.Unmarshal(dt, &img); err != nil { + debugLog(b, "failed to unmarshal image") + slog.Error("failed to unmarshal image", "FROM", err) + } + } else { + _, _, dt, err := gw.Client.ResolveImageConfig(c, b.ctx, baseImg, sourceresolver.Opt{ + ImageOpt: &sourceresolver.ResolveImageOpt{ + ResolveMode: llb.ResolveModeDefault.String(), + }, + }) + if err != nil { + debugLog(b, "failed to resolve image") + slog.Error("failed to resolve image", "FROM", err) + } + if err := json.Unmarshal(dt, &img); err != nil { + debugLog(b, "failed to unmarshal image") + slog.Error("failed to unmarshal image", "FROM", err) + } + + } b.state = llb.Image(stage.From) + b.image = img + + // initialize metadata + for _, env := range img.Config.Env { + debugLog(b, env) + k, v := parseKeyValue(env) + b.state = b.state.AddEnv(k, v) + } } for i := range *stage.Steps { - log.Printf("building stage %#v\n", (*stage.Steps)[i]) + slog.Info("Building steps", "stage", (*stage.Steps)[i]) b.state = (*stage.Steps)[i].Evaluate(b) } return b.state } -func (j *Jockerfile) ToLLB() llb.State { +func (j *Jockerfile) ToLLB(debug string, ctx context.Context, c client.Client) llb.State { b := BuildContext{ stages: make(map[string]llb.State), + debug: debug, + ctx: ctx, } var state llb.State opts := []llb.LocalOption{ @@ -94,11 +195,16 @@ func (j *Jockerfile) ToLLB() llb.State { b.context = llb.Local("context", opts...) for _, stage := range j.Stages { - log.Println("building stage", stage.Name) - - state = stage.ToLLB(&b) + slog.Info("Building stage", "ctx", stage.Name) + state = stage.ToLLB(&b, c) b.stages[stage.Name] = state } + // after all stages align the imageConfig to export + slog.Info("setting image config") + slog.Info(b.image.Config.WorkingDir) + + j.Image = b.image + return state }