Draft: see what the diff between ticktock and main is

cache/remotecache/import.go

@@ -302,8 +302,9 @@ type image struct {
 	Rootfs struct {
 		DiffIDs []digest.Digest `json:"diff_ids"`
 	} `json:"rootfs"`
-	Cache   []byte `json:"moby.buildkit.cache.v0"`
-	History []struct {
+	Cache              []byte `json:"moby.buildkit.cache.v0"`
+	EarthlyInlineCache []byte `json:"earthly.inlinecache.v0"`
+	History            []struct {
 		Created    *time.Time `json:"created,omitempty"`
 		CreatedBy  string     `json:"created_by,omitempty"`
 		EmptyLayer bool       `json:"empty_layer,omitempty"`

cache/remotecache/inline.go

@@ -0,0 +1,253 @@
+package remotecache
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/labels"
+	v1 "github.com/moby/buildkit/cache/remotecache/v1"
+	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/util/bklog"
+	"github.com/moby/buildkit/util/imageutil"
+	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/worker"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// earthlyInlineCacheItem stores a relation between a simple solver cache key &
+// a remote image descriptor. Used for inline caching.
+type earthlyInlineCacheItem struct {
+	Key        digest.Digest `json:"cacheKey"`
+	Descriptor digest.Digest `json:"descriptor"`
+}
+
+// EarthlyInlineCacheRemotes produces a map of cache keys to remote sources by
+// parsing inline-cache metadata from a remote image's config data.
+func EarthlyInlineCacheRemotes(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor, w worker.Worker) (map[digest.Digest]*solver.Remote, error) {
+	dt, err := readBlob(ctx, provider, desc)
+	if err != nil {
+		return nil, err
+	}
+
+	manifestType, err := imageutil.DetectManifestBlobMediaType(dt)
+	if err != nil {
+		return nil, err
+	}
+
+	layerDone := progress.OneOff(ctx, fmt.Sprintf("inferred cache manifest type: %s", manifestType))
+	layerDone(nil)
+
+	configDesc, err := configDescriptor(dt, manifestType)
+	if err != nil {
+		return nil, err
+	}
+
+	if configDesc.Digest != "" {
+		return nil, errors.New("expected empty digest value")
+	}
+
+	m := map[digest.Digest][]byte{}
+
+	if err := allDistributionManifests(ctx, provider, dt, m); err != nil {
+		return nil, err
+	}
+
+	remotes := map[digest.Digest]*solver.Remote{}
+
+	for _, dt := range m {
+		var m ocispecs.Manifest
+
+		if err := json.Unmarshal(dt, &m); err != nil {
+			return nil, errors.Wrap(err, "failed to unmarshal manifest")
+		}
+
+		if m.Config.Digest == "" || len(m.Layers) == 0 {
+			continue
+		}
+
+		p, err := content.ReadBlob(ctx, provider, m.Config)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read blob")
+		}
+
+		var img image
+
+		if err := json.Unmarshal(p, &img); err != nil {
+			return nil, errors.Wrap(err, "failed to unmarshal image")
+		}
+
+		if len(img.Rootfs.DiffIDs) != len(m.Layers) {
+			bklog.G(ctx).Warnf("invalid image with mismatching manifest and config")
+			continue
+		}
+
+		if img.EarthlyInlineCache == nil {
+			continue
+		}
+
+		cacheItems := []earthlyInlineCacheItem{}
+		if err := json.Unmarshal(img.EarthlyInlineCache, &cacheItems); err != nil {
+			return nil, errors.Wrap(err, "failed to unmarshal cache items")
+		}
+
+		layers, err := preprocessLayers(img, m)
+		if err != nil {
+			return nil, err
+		}
+
+		found := extractRemotes(provider, cacheItems, layers)
+		for key, remote := range found {
+			remotes[key] = remote
+		}
+	}
+
+	return remotes, nil
+}
+
+// extractRemotes constructs a list of descriptors--which represent the layer
+// chain for the given digest--for each of the items discovered in the inline
+// metadata.
+func extractRemotes(provider content.Provider, cacheItems []earthlyInlineCacheItem, layers []ocispecs.Descriptor) map[digest.Digest]*solver.Remote {
+
+	remotes := map[digest.Digest]*solver.Remote{}
+
+	for _, cacheItem := range cacheItems {
+		descs := []ocispecs.Descriptor{}
+
+		found := false
+		for _, layer := range layers {
+			descs = append(descs, layer)
+			if layer.Digest == cacheItem.Descriptor {
+				found = true
+				break
+			}
+		}
+
+		if found {
+			remote := &solver.Remote{
+				Descriptors: descs,
+				Provider:    provider,
+			}
+
+			remotes[cacheItem.Key] = remote
+		}
+	}
+
+	return remotes
+}
+
+// preprocessLayers adds custom annotations which are used later when
+// reconstructing the ref.
+func preprocessLayers(img image, m ocispecs.Manifest) ([]ocispecs.Descriptor, error) {
+	createdDates, createdMsg, err := parseCreatedLayerInfo(img)
+	if err != nil {
+		return nil, err
+	}
+
+	n := len(m.Layers)
+
+	if len(createdDates) != n {
+		return nil, errors.New("unexpected creation dates length")
+	}
+
+	if len(createdMsg) != n {
+		return nil, errors.New("unexpected creation messages length")
+	}
+
+	if len(img.Rootfs.DiffIDs) != n {
+		return nil, errors.New("unexpected rootfs diff IDs")
+	}
+
+	ret := []ocispecs.Descriptor{}
+
+	for i, layer := range m.Layers {
+		if layer.Annotations == nil {
+			layer.Annotations = map[string]string{}
+		}
+
+		if createdAt := createdDates[i]; createdAt != "" {
+			layer.Annotations["buildkit/createdat"] = createdAt
+		}
+
+		if createdBy := createdMsg[i]; createdBy != "" {
+			layer.Annotations["buildkit/description"] = createdBy
+		}
+
+		layer.Annotations[labels.LabelUncompressed] = img.Rootfs.DiffIDs[i].String()
+
+		ret = append(ret, layer)
+	}
+
+	return ret, nil
+}
+
+// configDescriptor parses and returns the correct manifest for the given manifest type.
+func configDescriptor(dt []byte, manifestType string) (ocispecs.Descriptor, error) {
+	var configDesc ocispecs.Descriptor
+
+	switch manifestType {
+	case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
+		var mfst ocispecs.Index
+		if err := json.Unmarshal(dt, &mfst); err != nil {
+			return ocispecs.Descriptor{}, err
+		}
+
+		for _, m := range mfst.Manifests {
+			if m.MediaType == v1.CacheConfigMediaTypeV0 {
+				configDesc = m
+				continue
+			}
+		}
+	case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest:
+		var mfst ocispecs.Manifest
+		if err := json.Unmarshal(dt, &mfst); err != nil {
+			return ocispecs.Descriptor{}, err
+		}
+
+		if mfst.Config.MediaType == v1.CacheConfigMediaTypeV0 {
+			configDesc = mfst.Config
+		}
+	default:
+		return ocispecs.Descriptor{}, errors.Errorf("unsupported or uninferrable manifest type %s", manifestType)
+	}
+
+	return configDesc, nil
+}
+
+// allDistributionManifests pulls all manifest data & linked manifests using the provider.
+func allDistributionManifests(ctx context.Context, provider content.Provider, dt []byte, m map[digest.Digest][]byte) error {
+	mt, err := imageutil.DetectManifestBlobMediaType(dt)
+	if err != nil {
+		return err
+	}
+
+	switch mt {
+	case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest:
+		m[digest.FromBytes(dt)] = dt
+	case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
+		var index ocispecs.Index
+		if err := json.Unmarshal(dt, &index); err != nil {
+			return errors.WithStack(err)
+		}
+
+		for _, d := range index.Manifests {
+			if _, ok := m[d.Digest]; ok {
+				continue
+			}
+			p, err := content.ReadBlob(ctx, provider, d)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			if err := allDistributionManifests(ctx, provider, p, m); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

cache/remotecache/registry/inline.go

@@ -0,0 +1,58 @@
+package registry
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/moby/buildkit/cache/remotecache"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/solver"
+	"github.com/moby/buildkit/util/contentutil"
+	"github.com/moby/buildkit/util/resolver"
+	"github.com/moby/buildkit/util/resolver/limited"
+	"github.com/moby/buildkit/worker"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+// EarthlyInlineCacheRemotes fetches a group of remote sources based on values
+// discovered in a remote image's inline-cache metadata field.
+func EarthlyInlineCacheRemotes(ctx context.Context, sm *session.Manager, w worker.Worker, hosts docker.RegistryHosts, g session.Group, attrs map[string]string) (map[digest.Digest]*solver.Remote, error) {
+	ref, err := canonicalizeRef(attrs[attrRef])
+	if err != nil {
+		return nil, err
+	}
+
+	refString := ref.String()
+
+	insecure := false
+	if v, ok := attrs[attrInsecure]; ok {
+		val, err := strconv.ParseBool(v)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse %s", attrInsecure)
+		}
+		insecure = val
+	}
+
+	scope, hosts := registryConfig(hosts, ref, "pull", insecure)
+	remote := resolver.DefaultPool.GetResolver(hosts, refString, scope, sm, g)
+
+	xref, desc, err := remote.Resolve(ctx, refString)
+	if err != nil {
+		return nil, err
+	}
+
+	fetcher, err := remote.Fetcher(ctx, xref)
+	if err != nil {
+		return nil, err
+	}
+
+	src := &withDistributionSourceLabel{
+		Provider: contentutil.FromFetcher(limited.Default.WrapFetcher(fetcher, refString)),
+		ref:      refString,
+		source:   w.ContentStore(),
+	}
+
+	return remotecache.EarthlyInlineCacheRemotes(ctx, src, desc, w)
+}

client/client.go

@@ -52,6 +52,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
 		grpc.WithDefaultCallOptions(grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(10*time.Millisecond, 0.1))), //earthly
 	}
 	needDialer := true
+	useDefaultDialer := false // earthly-specific
 
 	var unary []grpc.UnaryClientInterceptor
 	var stream []grpc.StreamClientInterceptor
@@ -94,6 +95,11 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
 			headersKV = h.kv
 		}
 
+		// earthly-specific
+		if _, ok := o.(*withDefaultGRPCDialer); ok {
+			useDefaultDialer = true
+		}
+
 		if opt, ok := o.(*withGRPCDialOption); ok {
 			customDialOptions = append(customDialOptions, opt.opt)
 		}
@@ -121,7 +127,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
 		stream = append(stream, otelgrpc.StreamClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators)))
 	}
 
-	if needDialer {
+	if needDialer && !useDefaultDialer {
 		dialFn, err := resolveDialer(address)
 		if err != nil {
 			return nil, err
@@ -164,6 +170,14 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error
 	gopts = append(gopts, grpc.WithChainStreamInterceptor(stream...))
 	gopts = append(gopts, customDialOptions...)
 
+	// earthly-specific
+	if useDefaultDialer {
+		split := strings.Split(address, "://")
+		if len(split) > 0 {
+			address = split[1]
+		}
+	}
+
 	conn, err := grpc.DialContext(ctx, address, gopts...)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to dial %q . make sure buildkitd is running", address)

client/client_earthly.go

@@ -32,3 +32,14 @@ func headersStreamInterceptor(kv ...string) grpc.StreamClientInterceptor {
 		return streamer(ctx, desc, cc, method, opts...)
 	}
 }
+
+// WithDefaultGRPCDialer triggers the internal gRPC dialer to be used instead of the buildkit default.
+// This can be important when buildkit server is behind an HTTP connect proxy,
+// since the default dialer in gRPC already knows how to use those.
+func WithDefaultGRPCDialer() ClientOpt {
+	return &withDefaultGRPCDialer{}
+}
+
+type withDefaultGRPCDialer struct{}
+
+func (*withDefaultGRPCDialer) isClientOpt() {}

cmd/buildkitd/main.go

@@ -843,6 +843,8 @@ func newController(c *cli.Context, cfg *config.Config, shutdownCh chan struct{})
 		LeaseManager:              w.LeaseManager(),
 		ContentStore:              w.ContentStore(),
 		HistoryConfig:             cfg.History,
+		RootDir:                   cfg.Root,
+		RegistryHosts:             resolverFn,
 	})
 }