From e318db52b989c31e5ebb21b57247d9c718c30100 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 12 Mar 2024 14:18:23 -0700 Subject: [PATCH 01/54] WIP --- solver/jobs.go | 101 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index d927352a2..ad6a43266 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -310,6 +310,7 @@ func (jl *Solver) getEdge(e Edge) *edge { } func (jl *Solver) subBuild(ctx context.Context, e Edge, parent Vertex) (CachedResult, error) { + // MH: Does not appear to be called in my tests v, err := jl.load(e.Vertex, parent, nil) if err != nil { return nil, err @@ -528,24 +529,108 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } } +type dumbBuilder struct { + resolveOpFunc ResolveOpFunc + solver *Solver +} + +func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { + + // Ordered list of vertices to build. + digests, vertices := b.exploreVertices(e) + + var ret CachedResult + + for i, d := range digests { + vertex, ok := vertices[d] + if !ok { + return nil, errors.Errorf("digest %s not found", d) + } + + defaultCache := NewInMemoryCacheManager() + + st := &state{ + opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: b.resolveOpFunc}, + jobs: map[*Job]struct{}{}, + parents: map[digest.Digest]struct{}{}, + childVtx: map[digest.Digest]struct{}{}, + allPw: map[progress.Writer]struct{}{}, + mpw: progress.NewMultiWriter(progress.WithMetadata("vertex", d)), + mspan: tracing.NewMultiSpan(), + vtx: vertex, + clientVertex: initClientVertex(vertex), + edges: map[Index]*edge{}, + index: nil, + mainCache: defaultCache, + cache: map[string]CacheManager{}, + solver: b.solver, + origDigest: vertex.Digest(), + } + + fmt.Println("Processing vertex", e.Vertex.Name()) + + edge := st.getEdge(Index(i)) + + cm, err := edge.op.CacheMap(ctx, i) + if err != nil { + return nil, err + } + + edge.cacheMap = cm.CacheMap + + res, err := edge.execOp(ctx) + if err != nil { + return nil, err + } + + ret = res.(CachedResult) + } + + return ret, nil +} + +func (b *dumbBuilder) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { + + digests := []digest.Digest{e.Vertex.Digest()} + vertices := map[digest.Digest]Vertex{ + e.Vertex.Digest(): e.Vertex, + } + + for _, edge := range e.Vertex.Inputs() { + d, v := b.exploreVertices(edge) + digests = append(d, digests...) + for key, value := range v { + vertices[key] = value + } + } + + return digests, vertices +} + func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { j.span = span } - v, err := j.list.load(e.Vertex, nil, j) + b := &dumbBuilder{resolveOpFunc: j.list.opts.ResolveOpFunc, solver: j.list} + res, err := b.build(ctx, e) if err != nil { return nil, err } - e.Vertex = v - res, err := j.list.s.build(ctx, e) - if err != nil { - return nil, err - } + // v, err := j.list.load(e.Vertex, nil, j) + // if err != nil { + // return nil, err + // } + // e.Vertex = v - j.list.mu.Lock() - defer j.list.mu.Unlock() + // res, err := j.list.s.build(ctx, e) + // if err != nil { + // return nil, err + // } + + // j.list.mu.Lock() + // defer j.list.mu.Unlock() return &withProvenance{CachedResult: res, j: j, e: e}, nil } From c2c3388a73047fd34d3b7f11b04fff69c762e54f Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 12 Mar 2024 20:22:05 -0700 Subject: [PATCH 02/54] WIP --- solver/jobs.go | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index ad6a43266..d0d0c7c81 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -532,6 +532,7 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { type dumbBuilder struct { resolveOpFunc ResolveOpFunc solver *Solver + job *Job } func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { @@ -539,9 +540,11 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { // Ordered list of vertices to build. digests, vertices := b.exploreVertices(e) + cache := map[string]CachedResult{} + var ret CachedResult - for i, d := range digests { + for _, d := range digests { vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) @@ -551,7 +554,6 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { st := &state{ opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: b.resolveOpFunc}, - jobs: map[*Job]struct{}{}, parents: map[digest.Digest]struct{}{}, childVtx: map[digest.Digest]struct{}{}, allPw: map[progress.Writer]struct{}{}, @@ -560,18 +562,32 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { vtx: vertex, clientVertex: initClientVertex(vertex), edges: map[Index]*edge{}, - index: nil, + index: b.solver.index, mainCache: defaultCache, cache: map[string]CacheManager{}, solver: b.solver, origDigest: vertex.Digest(), } + st.jobs = map[*Job]struct{}{ + b.job: {}, + } + fmt.Println("Processing vertex", e.Vertex.Name()) - edge := st.getEdge(Index(i)) + edge := st.getEdge(e.Index) - cm, err := edge.op.CacheMap(ctx, i) + edge.deps = make([]*dep, 0, len(edge.edge.Vertex.Inputs())) + inputs := edge.edge.Vertex.Inputs() + for i := range inputs { + dep := newDep(Index(i)) + if v, ok := cache[inputs[i].Vertex.Digest().String()]; ok { + dep.result = NewSharedCachedResult(v) + } + edge.deps = append(edge.deps, dep) + } + + cm, err := edge.op.CacheMap(ctx, int(e.Index)) if err != nil { return nil, err } @@ -583,7 +599,13 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { return nil, err } - ret = res.(CachedResult) + cachedResult := res.(CachedResult) + + edge.result = NewSharedCachedResult(cachedResult) + edge.state = edgeStatusComplete + + ret = cachedResult + cache[d.String()] = cachedResult } return ret, nil @@ -612,7 +634,11 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er j.span = span } - b := &dumbBuilder{resolveOpFunc: j.list.opts.ResolveOpFunc, solver: j.list} + b := &dumbBuilder{ + resolveOpFunc: j.list.opts.ResolveOpFunc, + solver: j.list, + job: j, + } res, err := b.build(ctx, e) if err != nil { return nil, err @@ -968,6 +994,8 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp, return nil, err } + fmt.Println(len(res), index) + if len(res) <= index { return s.CacheMap(ctx, index) } From e073523536f242861690888feccc7772068f4bd0 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 12 Mar 2024 21:34:30 -0700 Subject: [PATCH 03/54] Status --- solver/jobs.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index d0d0c7c81..8fb106db6 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -573,12 +573,12 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { b.job: {}, } - fmt.Println("Processing vertex", e.Vertex.Name()) + fmt.Println("Processing vertex", vertex.Name()) edge := st.getEdge(e.Index) - edge.deps = make([]*dep, 0, len(edge.edge.Vertex.Inputs())) - inputs := edge.edge.Vertex.Inputs() + edge.deps = make([]*dep, 0, len(vertex.Inputs())) + inputs := vertex.Inputs() for i := range inputs { dep := newDep(Index(i)) if v, ok := cache[inputs[i].Vertex.Digest().String()]; ok { @@ -601,6 +601,8 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { cachedResult := res.(CachedResult) + b.job.pw.Write(identity.NewID(), st.clientVertex) + edge.result = NewSharedCachedResult(cachedResult) edge.state = edgeStatusComplete @@ -994,8 +996,6 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp, return nil, err } - fmt.Println(len(res), index) - if len(res) <= index { return s.CacheMap(ctx, index) } From 94fced96530f479c7da5bfd32d51980053cb1f4d Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 14 Mar 2024 12:00:43 -0700 Subject: [PATCH 04/54] Cache --- solver/jobs.go | 23 ++++++++++++++++++++--- solver/scheduler.go | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 8fb106db6..d8517334f 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -535,13 +535,14 @@ type dumbBuilder struct { job *Job } +var cache = map[string]CachedResult{} +var mu = sync.Mutex{} + func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { // Ordered list of vertices to build. digests, vertices := b.exploreVertices(e) - cache := map[string]CachedResult{} - var ret CachedResult for _, d := range digests { @@ -573,7 +574,7 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { b.job: {}, } - fmt.Println("Processing vertex", vertex.Name()) + //fmt.Println("Processing vertex", vertex.Name(), d.String()) edge := st.getEdge(e.Index) @@ -594,6 +595,18 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { edge.cacheMap = cm.CacheMap + mu.Lock() + fmt.Println("Checking cache", d.String()) + if r, ok := cache[d.String()]; ok { + fmt.Println("Cached!") + st.clientVertex.Cached = true + b.job.pw.Write(identity.NewID(), st.clientVertex) + ret = r + mu.Unlock() + continue + } + mu.Unlock() + res, err := edge.execOp(ctx) if err != nil { return nil, err @@ -607,7 +620,11 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { edge.state = edgeStatusComplete ret = cachedResult + mu.Lock() + + fmt.Println("Adding cache!", d.String()) cache[d.String()] = cachedResult + mu.Unlock() } return ret, nil diff --git a/solver/scheduler.go b/solver/scheduler.go index d36815615..e462ef288 100644 --- a/solver/scheduler.go +++ b/solver/scheduler.go @@ -34,7 +34,7 @@ func newScheduler(ef edgeFactory) *scheduler { } s.cond = cond.NewStatefulCond(&s.mu) - go s.loop() + //go s.loop() return s } From 10527b4986ff5e3bebef77006392ee517060263f Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 14 Mar 2024 22:03:52 -0700 Subject: [PATCH 05/54] Write output & deduplicate --- solver/jobs.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index d8517334f..53afa515d 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -546,6 +546,7 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { var ret CachedResult for _, d := range digests { + mu.Lock() vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) @@ -574,6 +575,8 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { b.job: {}, } + st.mpw.Add(b.job.pw) + //fmt.Println("Processing vertex", vertex.Name(), d.String()) edge := st.getEdge(e.Index) @@ -595,17 +598,13 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { edge.cacheMap = cm.CacheMap - mu.Lock() - fmt.Println("Checking cache", d.String()) if r, ok := cache[d.String()]; ok { - fmt.Println("Cached!") st.clientVertex.Cached = true - b.job.pw.Write(identity.NewID(), st.clientVertex) + st.mpw.Write(identity.NewID(), st.clientVertex) ret = r mu.Unlock() continue } - mu.Unlock() res, err := edge.execOp(ctx) if err != nil { @@ -614,13 +613,12 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { cachedResult := res.(CachedResult) - b.job.pw.Write(identity.NewID(), st.clientVertex) + st.mpw.Write(identity.NewID(), st.clientVertex) edge.result = NewSharedCachedResult(cachedResult) edge.state = edgeStatusComplete ret = cachedResult - mu.Lock() fmt.Println("Adding cache!", d.String()) cache[d.String()] = cachedResult @@ -645,7 +643,16 @@ func (b *dumbBuilder) exploreVertices(e Edge) ([]digest.Digest, map[digest.Diges } } - return digests, vertices + ret := []digest.Digest{} + m := map[digest.Digest]struct{}{} + for _, d := range digests { + if _, ok := m[d]; !ok { + ret = append(ret, d) + m[d] = struct{}{} + } + } + + return ret, vertices } func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { From 9fa72212974fc0fad43137db7ed14fa7b2b80c88 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 17 Mar 2024 14:49:59 -0700 Subject: [PATCH 06/54] Remove edge code --- solver/jobs.go | 104 ++++++++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 53afa515d..747d3f830 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -529,33 +529,48 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } } -type dumbBuilder struct { +type simpleSolver struct { resolveOpFunc ResolveOpFunc solver *Solver job *Job } -var cache = map[string]CachedResult{} +var cache = map[string]Result{} +var inFlight = map[string]struct{}{} var mu = sync.Mutex{} -func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { +func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) { // Ordered list of vertices to build. - digests, vertices := b.exploreVertices(e) + digests, vertices := s.exploreVertices(e) - var ret CachedResult + var ret Result for _, d := range digests { - mu.Lock() vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) } + for { + mu.Lock() + _, ok := inFlight[d.String()] + mu.Unlock() + if ok { + time.Sleep(100 * time.Millisecond) + } else { + break + } + } + + mu.Lock() + inFlight[d.String()] = struct{}{} + mu.Unlock() + defaultCache := NewInMemoryCacheManager() st := &state{ - opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: b.resolveOpFunc}, + opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: s.resolveOpFunc}, parents: map[digest.Digest]struct{}{}, childVtx: map[digest.Digest]struct{}{}, allPw: map[progress.Writer]struct{}{}, @@ -564,71 +579,72 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { vtx: vertex, clientVertex: initClientVertex(vertex), edges: map[Index]*edge{}, - index: b.solver.index, + index: s.solver.index, mainCache: defaultCache, cache: map[string]CacheManager{}, - solver: b.solver, + solver: s.solver, origDigest: vertex.Digest(), } st.jobs = map[*Job]struct{}{ - b.job: {}, + s.job: {}, } - st.mpw.Add(b.job.pw) + st.mpw.Add(s.job.pw) - //fmt.Println("Processing vertex", vertex.Name(), d.String()) + notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - edge := st.getEdge(e.Index) - - edge.deps = make([]*dep, 0, len(vertex.Inputs())) - inputs := vertex.Inputs() - for i := range inputs { - dep := newDep(Index(i)) - if v, ok := cache[inputs[i].Vertex.Digest().String()]; ok { - dep.result = NewSharedCachedResult(v) - } - edge.deps = append(edge.deps, dep) + mu.Lock() + if v, ok := cache[d.String()]; ok { + delete(inFlight, d.String()) + mu.Unlock() + notifyCompleted(nil, true) + ret = v + continue } + mu.Unlock() - cm, err := edge.op.CacheMap(ctx, int(e.Index)) + op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) + + // CacheMap populates required fields in SourceOp. + _, err := op.CacheMap(ctx, int(e.Index)) if err != nil { return nil, err } - edge.cacheMap = cm.CacheMap + var inDigests []string + for _, in := range vertex.Inputs() { + inDigests = append(inDigests, in.Vertex.Digest().String()) + } - if r, ok := cache[d.String()]; ok { - st.clientVertex.Cached = true - st.mpw.Write(identity.NewID(), st.clientVertex) - ret = r - mu.Unlock() - continue + var inputs []Result + for _, in := range vertex.Inputs() { + if v, ok := cache[in.Vertex.Digest().String()]; ok { + inputs = append(inputs, v) + } } - res, err := edge.execOp(ctx) + results, _, err := op.Exec(ctx, inputs) if err != nil { + notifyCompleted(err, false) return nil, err } - cachedResult := res.(CachedResult) - - st.mpw.Write(identity.NewID(), st.clientVertex) + notifyCompleted(nil, false) - edge.result = NewSharedCachedResult(cachedResult) - edge.state = edgeStatusComplete + res := results[int(e.Index)] + ret = res - ret = cachedResult - - fmt.Println("Adding cache!", d.String()) - cache[d.String()] = cachedResult + mu.Lock() + delete(inFlight, d.String()) + cache[d.String()] = res mu.Unlock() } - return ret, nil + return NewCachedResult(ret, []ExportableCacheKey{}), nil } -func (b *dumbBuilder) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { +func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { digests := []digest.Digest{e.Vertex.Digest()} vertices := map[digest.Digest]Vertex{ @@ -636,7 +652,7 @@ func (b *dumbBuilder) exploreVertices(e Edge) ([]digest.Digest, map[digest.Diges } for _, edge := range e.Vertex.Inputs() { - d, v := b.exploreVertices(edge) + d, v := s.exploreVertices(edge) digests = append(d, digests...) for key, value := range v { vertices[key] = value @@ -660,7 +676,7 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er j.span = span } - b := &dumbBuilder{ + b := &simpleSolver{ resolveOpFunc: j.list.opts.ResolveOpFunc, solver: j.list, job: j, From 8a63a039abf96ad6e425cddbd7d55337c4acdf03 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 18 Mar 2024 09:27:39 -0700 Subject: [PATCH 07/54] Cache --- solver/jobs.go | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 747d3f830..14841b503 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -536,6 +536,8 @@ type simpleSolver struct { } var cache = map[string]Result{} +var cacheDigests = map[string]string{} + var inFlight = map[string]struct{}{} var mu = sync.Mutex{} @@ -594,32 +596,48 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - mu.Lock() - if v, ok := cache[d.String()]; ok { - delete(inFlight, d.String()) - mu.Unlock() - notifyCompleted(nil, true) - ret = v - continue - } - mu.Unlock() - op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) // CacheMap populates required fields in SourceOp. - _, err := op.CacheMap(ctx, int(e.Index)) + cm, err := op.CacheMap(ctx, int(e.Index)) if err != nil { return nil, err } - var inDigests []string + fmt.Println(vertex.Name()) + fmt.Println("CacheMap Digest:", cm.Digest) + fmt.Println("LLB Digest:", d) + for _, in := range vertex.Inputs() { - inDigests = append(inDigests, in.Vertex.Digest().String()) + fmt.Println("> input name", in.Vertex.Name()) + fmt.Println("> input digest", in.Vertex.Digest()) + } + + for _, in := range cm.Deps { + fmt.Println("> CacheMap selector:", in.Selector) } + fmt.Println("----") + + cacheDigest := cm.Digest.String() + mu.Lock() + cacheDigests[d.String()] = cacheDigest + mu.Unlock() + + mu.Lock() + if v, ok := cache[cacheDigest]; ok { + delete(inFlight, d.String()) + mu.Unlock() + notifyCompleted(nil, true) + ret = v + continue + } + mu.Unlock() + var inputs []Result for _, in := range vertex.Inputs() { - if v, ok := cache[in.Vertex.Digest().String()]; ok { + key := cacheDigests[in.Vertex.Digest().String()] + if v, ok := cache[key]; ok { inputs = append(inputs, v) } } @@ -637,7 +655,7 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) mu.Lock() delete(inFlight, d.String()) - cache[d.String()] = res + cache[cacheDigest] = res mu.Unlock() } From c778fd4c5459af967a54b04a73df8407b986e5d7 Mon Sep 17 00:00:00 2001 From: Alex Couture-Beil Date: Fri, 15 Mar 2024 15:45:15 -0700 Subject: [PATCH 08/54] Support parallel build calls --- solver/jobs.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 53afa515d..477cddc8d 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -538,6 +538,8 @@ type dumbBuilder struct { var cache = map[string]CachedResult{} var mu = sync.Mutex{} +var execInProgress = map[string]struct{}{} + func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { // Ordered list of vertices to build. @@ -546,12 +548,25 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { var ret CachedResult for _, d := range digests { - mu.Lock() vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) } + dgst := d.String() + mu.Lock() + // TODO: replace busy-wait loop with a wait-for-channel-to-close approach + for { + if _, shouldWait := execInProgress[dgst]; !shouldWait { + execInProgress[dgst] = struct{}{} + mu.Unlock() + break + } + mu.Unlock() + time.Sleep(time.Millisecond * 10) + mu.Lock() + } + defaultCache := NewInMemoryCacheManager() st := &state{ @@ -598,16 +613,21 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { edge.cacheMap = cm.CacheMap - if r, ok := cache[d.String()]; ok { + if r, ok := cache[dgst]; ok { st.clientVertex.Cached = true st.mpw.Write(identity.NewID(), st.clientVertex) ret = r + mu.Lock() + delete(execInProgress, dgst) mu.Unlock() continue } res, err := edge.execOp(ctx) if err != nil { + mu.Lock() + delete(execInProgress, dgst) + mu.Unlock() return nil, err } @@ -620,8 +640,9 @@ func (b *dumbBuilder) build(ctx context.Context, e Edge) (CachedResult, error) { ret = cachedResult - fmt.Println("Adding cache!", d.String()) - cache[d.String()] = cachedResult + mu.Lock() + cache[dgst] = cachedResult + delete(execInProgress, dgst) mu.Unlock() } From 676524c66a37508d3dffd6ee5305766ee0920b27 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 19 Mar 2024 10:21:53 -0700 Subject: [PATCH 09/54] Generate consistent cache keys --- solver/jobs.go | 137 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 111 insertions(+), 26 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 14841b503..ddc37961b 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -2,10 +2,14 @@ package solver import ( "context" + "crypto/sha256" "fmt" + "hash" + "io" "sync" "time" + "github.com/davecgh/go-spew/spew" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" @@ -536,11 +540,21 @@ type simpleSolver struct { } var cache = map[string]Result{} -var cacheDigests = map[string]string{} - +var cacheMaps = map[string]*simpleCacheMap{} var inFlight = map[string]struct{}{} var mu = sync.Mutex{} +type cacheMapDep struct { + selector string + computed string +} + +type simpleCacheMap struct { + digest string + inputs []string + deps []cacheMapDep +} + func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) { // Ordered list of vertices to build. @@ -605,27 +619,61 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) } fmt.Println(vertex.Name()) - fmt.Println("CacheMap Digest:", cm.Digest) - fmt.Println("LLB Digest:", d) - - for _, in := range vertex.Inputs() { - fmt.Println("> input name", in.Vertex.Name()) - fmt.Println("> input digest", in.Vertex.Digest()) + fmt.Println("LLB digest:", d) + fmt.Println("CacheMap digest:", cm.Digest) + + // TODO: handle cm.Opts (CacheOpts) + scm := simpleCacheMap{ + digest: cm.Digest.String(), + deps: make([]cacheMapDep, len(cm.Deps)), + inputs: make([]string, len(cm.Deps)), } - - for _, in := range cm.Deps { - fmt.Println("> CacheMap selector:", in.Selector) + var inputs []Result + for i, in := range vertex.Inputs() { + cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) + if err != nil { + return nil, err + } + mu.Lock() + res, ok := cache[cacheKey] + mu.Unlock() + if !ok { + return nil, errors.Errorf("cache key not found: %s", cacheKey) + } + dep := cm.Deps[i] + spew.Dump(dep) + if dep.PreprocessFunc != nil { + err = dep.PreprocessFunc(ctx, res, st) + if err != nil { + return nil, err + } + } + scm.deps[i] = cacheMapDep{ + selector: dep.Selector.String(), + } + if dep.ComputeDigestFunc != nil { + compDigest, err := dep.ComputeDigestFunc(ctx, res, st) + if err != nil { + return nil, err + } + scm.deps[i].computed = compDigest.String() + } + scm.inputs[i] = in.Vertex.Digest().String() + inputs = append(inputs, res) } - - fmt.Println("----") - - cacheDigest := cm.Digest.String() mu.Lock() - cacheDigests[d.String()] = cacheDigest + cacheMaps[d.String()] = &scm mu.Unlock() + cacheKey, err := s.cacheKey(ctx, d.String()) + if err != nil { + return nil, err + } + + fmt.Println("Cache key", cacheKey) + mu.Lock() - if v, ok := cache[cacheDigest]; ok { + if v, ok := cache[cacheKey]; ok && v != nil { delete(inFlight, d.String()) mu.Unlock() notifyCompleted(nil, true) @@ -634,14 +682,6 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) } mu.Unlock() - var inputs []Result - for _, in := range vertex.Inputs() { - key := cacheDigests[in.Vertex.Digest().String()] - if v, ok := cache[key]; ok { - inputs = append(inputs, v) - } - } - results, _, err := op.Exec(ctx, inputs) if err != nil { notifyCompleted(err, false) @@ -650,18 +690,63 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) notifyCompleted(nil, false) + fmt.Println("Result length:", len(results)) + res := results[int(e.Index)] ret = res mu.Lock() delete(inFlight, d.String()) - cache[cacheDigest] = res + cache[cacheKey] = res mu.Unlock() } return NewCachedResult(ret, []ExportableCacheKey{}), nil } +func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { + h := sha256.New() + + fmt.Println("Computing cache key") + + err := s.calcCacheKey(ctx, d, h) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) error { + mu.Lock() + c, ok := cacheMaps[d] + mu.Unlock() + if !ok { + return errors.New("missing cache map key") + } + + spew.Dump(c) + + for _, in := range c.inputs { + err := s.calcCacheKey(ctx, in, h) + if err != nil { + return err + } + } + + io.WriteString(h, c.digest) + for _, dep := range c.deps { + if dep.selector != "" { + io.WriteString(h, dep.selector) + } + if dep.computed != "" { + io.WriteString(h, dep.computed) + } + } + + return nil +} + func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { digests := []digest.Digest{e.Vertex.Digest()} From 392d0ad7142c40b04a17b7fd1cedb5047cc8f2d2 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 19 Mar 2024 11:16:32 -0700 Subject: [PATCH 10/54] Refactoring --- solver/jobs.go | 207 ++++++++++++++++++++++++++++--------------------- 1 file changed, 120 insertions(+), 87 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index ddc37961b..d6f53f0af 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/davecgh/go-spew/spew" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" @@ -541,7 +540,7 @@ type simpleSolver struct { var cache = map[string]Result{} var cacheMaps = map[string]*simpleCacheMap{} -var inFlight = map[string]struct{}{} +var execInProgress = map[string]struct{}{} var mu = sync.Mutex{} type cacheMapDep struct { @@ -563,50 +562,27 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) var ret Result for _, d := range digests { + fmt.Println() + vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) } + mu.Lock() + // TODO: replace busy-wait loop with a wait-for-channel-to-close approach for { - mu.Lock() - _, ok := inFlight[d.String()] - mu.Unlock() - if ok { - time.Sleep(100 * time.Millisecond) - } else { + if _, shouldWait := execInProgress[d.String()]; !shouldWait { + execInProgress[d.String()] = struct{}{} + mu.Unlock() break } + mu.Unlock() + time.Sleep(time.Millisecond * 10) + mu.Lock() } - mu.Lock() - inFlight[d.String()] = struct{}{} - mu.Unlock() - - defaultCache := NewInMemoryCacheManager() - - st := &state{ - opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: s.resolveOpFunc}, - parents: map[digest.Digest]struct{}{}, - childVtx: map[digest.Digest]struct{}{}, - allPw: map[progress.Writer]struct{}{}, - mpw: progress.NewMultiWriter(progress.WithMetadata("vertex", d)), - mspan: tracing.NewMultiSpan(), - vtx: vertex, - clientVertex: initClientVertex(vertex), - edges: map[Index]*edge{}, - index: s.solver.index, - mainCache: defaultCache, - cache: map[string]CacheManager{}, - solver: s.solver, - origDigest: vertex.Digest(), - } - - st.jobs = map[*Job]struct{}{ - s.job: {}, - } - - st.mpw.Add(s.job.pw) + st := s.createState(vertex) notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) @@ -619,62 +595,25 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) } fmt.Println(vertex.Name()) - fmt.Println("LLB digest:", d) + fmt.Println("LLB digest:", d.String()) fmt.Println("CacheMap digest:", cm.Digest) - // TODO: handle cm.Opts (CacheOpts) - scm := simpleCacheMap{ - digest: cm.Digest.String(), - deps: make([]cacheMapDep, len(cm.Deps)), - inputs: make([]string, len(cm.Deps)), - } - var inputs []Result - for i, in := range vertex.Inputs() { - cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) - if err != nil { - return nil, err - } - mu.Lock() - res, ok := cache[cacheKey] - mu.Unlock() - if !ok { - return nil, errors.Errorf("cache key not found: %s", cacheKey) - } - dep := cm.Deps[i] - spew.Dump(dep) - if dep.PreprocessFunc != nil { - err = dep.PreprocessFunc(ctx, res, st) - if err != nil { - return nil, err - } - } - scm.deps[i] = cacheMapDep{ - selector: dep.Selector.String(), - } - if dep.ComputeDigestFunc != nil { - compDigest, err := dep.ComputeDigestFunc(ctx, res, st) - if err != nil { - return nil, err - } - scm.deps[i].computed = compDigest.String() - } - scm.inputs[i] = in.Vertex.Digest().String() - inputs = append(inputs, res) + inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) + if err != nil { + return nil, err } - mu.Lock() - cacheMaps[d.String()] = &scm - mu.Unlock() cacheKey, err := s.cacheKey(ctx, d.String()) if err != nil { return nil, err } - fmt.Println("Cache key", cacheKey) + fmt.Println("Computed cache key:", cacheKey) mu.Lock() if v, ok := cache[cacheKey]; ok && v != nil { - delete(inFlight, d.String()) + fmt.Println("Cache hit!") + delete(execInProgress, d.String()) mu.Unlock() notifyCompleted(nil, true) ret = v @@ -684,19 +623,20 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) results, _, err := op.Exec(ctx, inputs) if err != nil { + mu.Lock() + delete(execInProgress, d.String()) + mu.Unlock() notifyCompleted(err, false) return nil, err } notifyCompleted(nil, false) - fmt.Println("Result length:", len(results)) - res := results[int(e.Index)] ret = res mu.Lock() - delete(inFlight, d.String()) + delete(execInProgress, d.String()) cache[cacheKey] = res mu.Unlock() } @@ -704,11 +644,106 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return NewCachedResult(ret, []ExportableCacheKey{}), nil } +func (s *simpleSolver) createState(vertex Vertex) *state { + defaultCache := NewInMemoryCacheManager() + + st := &state{ + opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: s.resolveOpFunc}, + parents: map[digest.Digest]struct{}{}, + childVtx: map[digest.Digest]struct{}{}, + allPw: map[progress.Writer]struct{}{}, + mpw: progress.NewMultiWriter(progress.WithMetadata("vertex", vertex.Digest())), + mspan: tracing.NewMultiSpan(), + vtx: vertex, + clientVertex: initClientVertex(vertex), + edges: map[Index]*edge{}, + index: s.solver.index, + mainCache: defaultCache, + cache: map[string]CacheManager{}, + solver: s.solver, + origDigest: vertex.Digest(), + } + + st.jobs = map[*Job]struct{}{ + s.job: {}, + } + + st.mpw.Add(s.job.pw) + + return st +} + +func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { + // This struct is used to reconstruct a cache key from an LLB digest & all + // parents using consistent digests that depend on the full dependency chain. + // TODO: handle cm.Opts (CacheOpts)? + scm := simpleCacheMap{ + digest: cm.Digest.String(), + deps: make([]cacheMapDep, len(cm.Deps)), + inputs: make([]string, len(cm.Deps)), + } + + var inputs []Result + + for i, in := range vertex.Inputs() { + // Compute a cache key given the LLB digest value. + cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) + if err != nil { + return nil, err + } + + // Lookup the result for that cache key. + mu.Lock() + res, ok := cache[cacheKey] + mu.Unlock() + if !ok { + return nil, errors.Errorf("cache key not found: %s", cacheKey) + } + + dep := cm.Deps[i] + + // Unlazy the result. + if dep.PreprocessFunc != nil { + err = dep.PreprocessFunc(ctx, res, st) + if err != nil { + return nil, err + } + } + + // Add selectors (usually file references) to the struct. + scm.deps[i] = cacheMapDep{ + selector: dep.Selector.String(), + } + + // ComputeDigestFunc will usually checksum files. This is then used as + // part of the cache key to ensure it's consistent & distinct for this + // operation. + if dep.ComputeDigestFunc != nil { + compDigest, err := dep.ComputeDigestFunc(ctx, res, st) + if err != nil { + return nil, err + } + scm.deps[i].computed = compDigest.String() + } + + // Add input references to the struct as to link dependencies. + scm.inputs[i] = in.Vertex.Digest().String() + + // Add the cached result to the input set. These inputs are used to + // reconstruct dependencies (mounts, etc.) for a new container run. + inputs = append(inputs, res) + } + + mu.Lock() + cacheMaps[vertex.Digest().String()] = &scm + mu.Unlock() + + return inputs, nil +} + func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { h := sha256.New() - fmt.Println("Computing cache key") - err := s.calcCacheKey(ctx, d, h) if err != nil { return "", err @@ -725,8 +760,6 @@ func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) return errors.New("missing cache map key") } - spew.Dump(c) - for _, in := range c.inputs { err := s.calcCacheKey(ctx, in, h) if err != nil { From 0f5a46a08a03f4b6b04eab28ca738a02bd62dbb0 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 19 Mar 2024 16:20:56 -0700 Subject: [PATCH 11/54] Fix panic & move to simple.go --- solver/jobs.go | 278 -------------------------------------------- solver/simple.go | 294 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 294 insertions(+), 278 deletions(-) create mode 100644 solver/simple.go diff --git a/solver/jobs.go b/solver/jobs.go index d6f53f0af..293e29f0c 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -2,10 +2,7 @@ package solver import ( "context" - "crypto/sha256" "fmt" - "hash" - "io" "sync" "time" @@ -532,281 +529,6 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } } -type simpleSolver struct { - resolveOpFunc ResolveOpFunc - solver *Solver - job *Job -} - -var cache = map[string]Result{} -var cacheMaps = map[string]*simpleCacheMap{} -var execInProgress = map[string]struct{}{} -var mu = sync.Mutex{} - -type cacheMapDep struct { - selector string - computed string -} - -type simpleCacheMap struct { - digest string - inputs []string - deps []cacheMapDep -} - -func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) { - - // Ordered list of vertices to build. - digests, vertices := s.exploreVertices(e) - - var ret Result - - for _, d := range digests { - fmt.Println() - - vertex, ok := vertices[d] - if !ok { - return nil, errors.Errorf("digest %s not found", d) - } - - mu.Lock() - // TODO: replace busy-wait loop with a wait-for-channel-to-close approach - for { - if _, shouldWait := execInProgress[d.String()]; !shouldWait { - execInProgress[d.String()] = struct{}{} - mu.Unlock() - break - } - mu.Unlock() - time.Sleep(time.Millisecond * 10) - mu.Lock() - } - - st := s.createState(vertex) - - notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - - op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) - - // CacheMap populates required fields in SourceOp. - cm, err := op.CacheMap(ctx, int(e.Index)) - if err != nil { - return nil, err - } - - fmt.Println(vertex.Name()) - fmt.Println("LLB digest:", d.String()) - fmt.Println("CacheMap digest:", cm.Digest) - - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) - if err != nil { - return nil, err - } - - cacheKey, err := s.cacheKey(ctx, d.String()) - if err != nil { - return nil, err - } - - fmt.Println("Computed cache key:", cacheKey) - - mu.Lock() - if v, ok := cache[cacheKey]; ok && v != nil { - fmt.Println("Cache hit!") - delete(execInProgress, d.String()) - mu.Unlock() - notifyCompleted(nil, true) - ret = v - continue - } - mu.Unlock() - - results, _, err := op.Exec(ctx, inputs) - if err != nil { - mu.Lock() - delete(execInProgress, d.String()) - mu.Unlock() - notifyCompleted(err, false) - return nil, err - } - - notifyCompleted(nil, false) - - res := results[int(e.Index)] - ret = res - - mu.Lock() - delete(execInProgress, d.String()) - cache[cacheKey] = res - mu.Unlock() - } - - return NewCachedResult(ret, []ExportableCacheKey{}), nil -} - -func (s *simpleSolver) createState(vertex Vertex) *state { - defaultCache := NewInMemoryCacheManager() - - st := &state{ - opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: s.resolveOpFunc}, - parents: map[digest.Digest]struct{}{}, - childVtx: map[digest.Digest]struct{}{}, - allPw: map[progress.Writer]struct{}{}, - mpw: progress.NewMultiWriter(progress.WithMetadata("vertex", vertex.Digest())), - mspan: tracing.NewMultiSpan(), - vtx: vertex, - clientVertex: initClientVertex(vertex), - edges: map[Index]*edge{}, - index: s.solver.index, - mainCache: defaultCache, - cache: map[string]CacheManager{}, - solver: s.solver, - origDigest: vertex.Digest(), - } - - st.jobs = map[*Job]struct{}{ - s.job: {}, - } - - st.mpw.Add(s.job.pw) - - return st -} - -func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { - // This struct is used to reconstruct a cache key from an LLB digest & all - // parents using consistent digests that depend on the full dependency chain. - // TODO: handle cm.Opts (CacheOpts)? - scm := simpleCacheMap{ - digest: cm.Digest.String(), - deps: make([]cacheMapDep, len(cm.Deps)), - inputs: make([]string, len(cm.Deps)), - } - - var inputs []Result - - for i, in := range vertex.Inputs() { - // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) - if err != nil { - return nil, err - } - - // Lookup the result for that cache key. - mu.Lock() - res, ok := cache[cacheKey] - mu.Unlock() - if !ok { - return nil, errors.Errorf("cache key not found: %s", cacheKey) - } - - dep := cm.Deps[i] - - // Unlazy the result. - if dep.PreprocessFunc != nil { - err = dep.PreprocessFunc(ctx, res, st) - if err != nil { - return nil, err - } - } - - // Add selectors (usually file references) to the struct. - scm.deps[i] = cacheMapDep{ - selector: dep.Selector.String(), - } - - // ComputeDigestFunc will usually checksum files. This is then used as - // part of the cache key to ensure it's consistent & distinct for this - // operation. - if dep.ComputeDigestFunc != nil { - compDigest, err := dep.ComputeDigestFunc(ctx, res, st) - if err != nil { - return nil, err - } - scm.deps[i].computed = compDigest.String() - } - - // Add input references to the struct as to link dependencies. - scm.inputs[i] = in.Vertex.Digest().String() - - // Add the cached result to the input set. These inputs are used to - // reconstruct dependencies (mounts, etc.) for a new container run. - inputs = append(inputs, res) - } - - mu.Lock() - cacheMaps[vertex.Digest().String()] = &scm - mu.Unlock() - - return inputs, nil -} - -func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { - h := sha256.New() - - err := s.calcCacheKey(ctx, d, h) - if err != nil { - return "", err - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - -func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) error { - mu.Lock() - c, ok := cacheMaps[d] - mu.Unlock() - if !ok { - return errors.New("missing cache map key") - } - - for _, in := range c.inputs { - err := s.calcCacheKey(ctx, in, h) - if err != nil { - return err - } - } - - io.WriteString(h, c.digest) - for _, dep := range c.deps { - if dep.selector != "" { - io.WriteString(h, dep.selector) - } - if dep.computed != "" { - io.WriteString(h, dep.computed) - } - } - - return nil -} - -func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { - - digests := []digest.Digest{e.Vertex.Digest()} - vertices := map[digest.Digest]Vertex{ - e.Vertex.Digest(): e.Vertex, - } - - for _, edge := range e.Vertex.Inputs() { - d, v := s.exploreVertices(edge) - digests = append(d, digests...) - for key, value := range v { - vertices[key] = value - } - } - - ret := []digest.Digest{} - m := map[digest.Digest]struct{}{} - for _, d := range digests { - if _, ok := m[d]; !ok { - ret = append(ret, d) - m[d] = struct{}{} - } - } - - return ret, vertices -} - func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { j.span = span diff --git a/solver/simple.go b/solver/simple.go new file mode 100644 index 000000000..8c8874e7e --- /dev/null +++ b/solver/simple.go @@ -0,0 +1,294 @@ +package solver + +import ( + "context" + "crypto/sha256" + "fmt" + "hash" + "io" + "sync" + "time" + + "github.com/moby/buildkit/util/progress" + "github.com/moby/buildkit/util/tracing" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type simpleSolver struct { + resolveOpFunc ResolveOpFunc + solver *Solver + job *Job +} + +var cache = map[string]Result{} +var cacheMaps = map[string]*simpleCacheMap{} +var execInProgress = map[string]struct{}{} +var mu = sync.Mutex{} + +type cacheMapDep struct { + selector string + computed string +} + +type simpleCacheMap struct { + digest string + inputs []string + deps []cacheMapDep +} + +func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) { + + // Ordered list of vertices to build. + digests, vertices := s.exploreVertices(e) + + var ret Result + + for _, d := range digests { + fmt.Println() + + vertex, ok := vertices[d] + if !ok { + return nil, errors.Errorf("digest %s not found", d) + } + + mu.Lock() + // TODO: replace busy-wait loop with a wait-for-channel-to-close approach + for { + if _, shouldWait := execInProgress[d.String()]; !shouldWait { + execInProgress[d.String()] = struct{}{} + mu.Unlock() + break + } + mu.Unlock() + time.Sleep(time.Millisecond * 10) + mu.Lock() + } + + st := s.createState(vertex) + + notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) + + op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) + + // Required to access cache map results on state. + st.op = op + + // CacheMap populates required fields in SourceOp. + cm, err := op.CacheMap(ctx, int(e.Index)) + if err != nil { + return nil, err + } + + fmt.Println(vertex.Name()) + fmt.Println("LLB digest:", d.String()) + fmt.Println("CacheMap digest:", cm.Digest) + + inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) + if err != nil { + return nil, err + } + + cacheKey, err := s.cacheKey(ctx, d.String()) + if err != nil { + return nil, err + } + + fmt.Println("Computed cache key:", cacheKey) + + mu.Lock() + if v, ok := cache[cacheKey]; ok && v != nil { + fmt.Println("Cache hit!") + delete(execInProgress, d.String()) + mu.Unlock() + notifyCompleted(nil, true) + ret = v + continue + } + mu.Unlock() + + results, _, err := op.Exec(ctx, inputs) + if err != nil { + mu.Lock() + delete(execInProgress, d.String()) + mu.Unlock() + notifyCompleted(err, false) + return nil, err + } + + notifyCompleted(nil, false) + + res := results[int(e.Index)] + ret = res + + mu.Lock() + delete(execInProgress, d.String()) + cache[cacheKey] = res + mu.Unlock() + } + + return NewCachedResult(ret, []ExportableCacheKey{}), nil +} + +func (s *simpleSolver) createState(vertex Vertex) *state { + defaultCache := NewInMemoryCacheManager() + + st := &state{ + opts: SolverOpt{DefaultCache: defaultCache, ResolveOpFunc: s.resolveOpFunc}, + parents: map[digest.Digest]struct{}{}, + childVtx: map[digest.Digest]struct{}{}, + allPw: map[progress.Writer]struct{}{}, + mpw: progress.NewMultiWriter(progress.WithMetadata("vertex", vertex.Digest())), + mspan: tracing.NewMultiSpan(), + vtx: vertex, + clientVertex: initClientVertex(vertex), + edges: map[Index]*edge{}, + index: s.solver.index, + mainCache: defaultCache, + cache: map[string]CacheManager{}, + solver: s.solver, + origDigest: vertex.Digest(), + } + + st.jobs = map[*Job]struct{}{ + s.job: {}, + } + + st.mpw.Add(s.job.pw) + + return st +} + +func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { + // This struct is used to reconstruct a cache key from an LLB digest & all + // parents using consistent digests that depend on the full dependency chain. + // TODO: handle cm.Opts (CacheOpts)? + scm := simpleCacheMap{ + digest: cm.Digest.String(), + deps: make([]cacheMapDep, len(cm.Deps)), + inputs: make([]string, len(cm.Deps)), + } + + var inputs []Result + + for i, in := range vertex.Inputs() { + // Compute a cache key given the LLB digest value. + cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) + if err != nil { + return nil, err + } + + // Lookup the result for that cache key. + mu.Lock() + res, ok := cache[cacheKey] + mu.Unlock() + if !ok { + return nil, errors.Errorf("cache key not found: %s", cacheKey) + } + + dep := cm.Deps[i] + + // Unlazy the result. + if dep.PreprocessFunc != nil { + err = dep.PreprocessFunc(ctx, res, st) + if err != nil { + return nil, err + } + } + + // Add selectors (usually file references) to the struct. + scm.deps[i] = cacheMapDep{ + selector: dep.Selector.String(), + } + + // ComputeDigestFunc will usually checksum files. This is then used as + // part of the cache key to ensure it's consistent & distinct for this + // operation. + if dep.ComputeDigestFunc != nil { + compDigest, err := dep.ComputeDigestFunc(ctx, res, st) + if err != nil { + return nil, err + } + scm.deps[i].computed = compDigest.String() + } + + // Add input references to the struct as to link dependencies. + scm.inputs[i] = in.Vertex.Digest().String() + + // Add the cached result to the input set. These inputs are used to + // reconstruct dependencies (mounts, etc.) for a new container run. + inputs = append(inputs, res) + } + + mu.Lock() + cacheMaps[vertex.Digest().String()] = &scm + mu.Unlock() + + return inputs, nil +} + +func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { + h := sha256.New() + + err := s.calcCacheKey(ctx, d, h) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) error { + mu.Lock() + c, ok := cacheMaps[d] + mu.Unlock() + if !ok { + return errors.New("missing cache map key") + } + + for _, in := range c.inputs { + err := s.calcCacheKey(ctx, in, h) + if err != nil { + return err + } + } + + io.WriteString(h, c.digest) + for _, dep := range c.deps { + if dep.selector != "" { + io.WriteString(h, dep.selector) + } + if dep.computed != "" { + io.WriteString(h, dep.computed) + } + } + + return nil +} + +func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { + + digests := []digest.Digest{e.Vertex.Digest()} + vertices := map[digest.Digest]Vertex{ + e.Vertex.Digest(): e.Vertex, + } + + for _, edge := range e.Vertex.Inputs() { + d, v := s.exploreVertices(edge) + digests = append(d, digests...) + for key, value := range v { + vertices[key] = value + } + } + + ret := []digest.Digest{} + m := map[digest.Digest]struct{}{} + for _, d := range digests { + if _, ok := m[d]; !ok { + ret = append(ret, d) + m[d] = struct{}{} + } + } + + return ret, vertices +} From c3669c3b947352b61ff942f894f1c6cd6ff8f0c8 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 20 Mar 2024 15:39:14 -0700 Subject: [PATCH 12/54] Fix cached progress messages, hack local source --- solver/simple.go | 23 ++++++++--------------- source/local/source.go | 6 ++++++ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 8c8874e7e..87f008f61 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -67,8 +67,6 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) st := s.createState(vertex) - notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) // Required to access cache map results on state. @@ -80,10 +78,6 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return nil, err } - fmt.Println(vertex.Name()) - fmt.Println("LLB digest:", d.String()) - fmt.Println("CacheMap digest:", cm.Digest) - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) if err != nil { return nil, err @@ -94,13 +88,12 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return nil, err } - fmt.Println("Computed cache key:", cacheKey) - mu.Lock() if v, ok := cache[cacheKey]; ok && v != nil { - fmt.Println("Cache hit!") delete(execInProgress, d.String()) mu.Unlock() + ctx = progress.WithProgress(ctx, st.mpw) + notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) notifyCompleted(nil, true) ret = v continue @@ -112,12 +105,9 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) mu.Lock() delete(execInProgress, d.String()) mu.Unlock() - notifyCompleted(err, false) return nil, err } - notifyCompleted(nil, false) - res := results[int(e.Index)] ret = res @@ -130,6 +120,7 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return NewCachedResult(ret, []ExportableCacheKey{}), nil } +// createState creates a new state struct with required and placeholder values. func (s *simpleSolver) createState(vertex Vertex) *state { defaultCache := NewInMemoryCacheManager() @@ -227,10 +218,12 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V return inputs, nil } +// cacheKey recursively generates a cache key based on a sequence of ancestor +// operations & their cacheable values. func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { h := sha256.New() - err := s.calcCacheKey(ctx, d, h) + err := s.cacheKeyRecurse(ctx, d, h) if err != nil { return "", err } @@ -238,7 +231,7 @@ func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) error { +func (s *simpleSolver) cacheKeyRecurse(ctx context.Context, d string, h hash.Hash) error { mu.Lock() c, ok := cacheMaps[d] mu.Unlock() @@ -247,7 +240,7 @@ func (s *simpleSolver) calcCacheKey(ctx context.Context, d string, h hash.Hash) } for _, in := range c.inputs { - err := s.calcCacheKey(ctx, in, h) + err := s.cacheKeyRecurse(ctx, in, h) if err != nil { return err } diff --git a/source/local/source.go b/source/local/source.go index ae480be2e..0fd41c03a 100644 --- a/source/local/source.go +++ b/source/local/source.go @@ -123,6 +123,12 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, ind } sessionID = id } + + // Hack: The encoded session ID here is breaking the simplified caching + // approach in "simple.go". However, a consistent value here is likely + // unreliable with multiple users. Figure out another option. + sessionID = "session-id" + dt, err := json.Marshal(struct { SessionID string IncludePatterns []string From 29c7ae6b6a7d2e7a59f304f765f9f52cd4c6939f Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 21 Mar 2024 13:37:57 -0700 Subject: [PATCH 13/54] Remove extra print --- solver/simple.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 87f008f61..51402e484 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -45,8 +45,6 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) var ret Result for _, d := range digests { - fmt.Println() - vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) From fd93c592c387ca370790260e9fb00646f7708f2d Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 21 Mar 2024 16:19:01 -0700 Subject: [PATCH 14/54] Remove globals & refactor into structs --- solver/jobs.go | 20 ++-- solver/simple.go | 243 +++++++++++++++++++++++++++++++---------------- 2 files changed, 177 insertions(+), 86 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 293e29f0c..665ebebb3 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -42,6 +42,8 @@ type Solver struct { updateCond *sync.Cond s *scheduler index *edgeIndex + + simple *simpleSolver } type state struct { @@ -268,6 +270,13 @@ func NewSolver(opts SolverOpt) *Solver { opts: opts, index: newEdgeIndex(), } + + // TODO: This should be hoisted up a few layers as not to be bound to the + // original solver. For now, we just need a convenient place to initialize + // it once. + simple := newSimpleSolver(opts.ResolveOpFunc, jl) + jl.simple = simple + jl.s = newScheduler(jl) jl.updateCond = sync.NewCond(jl.mu.RLocker()) return jl @@ -534,16 +543,14 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er j.span = span } - b := &simpleSolver{ - resolveOpFunc: j.list.opts.ResolveOpFunc, - solver: j.list, - job: j, - } - res, err := b.build(ctx, e) + // TODO: Separate the new solver code from the original Solver & Job code. + res, err := j.list.simple.build(ctx, j, e) if err != nil { return nil, err } + // MH: Previous solver code is disabled below. + // v, err := j.list.load(e.Vertex, nil, j) // if err != nil { // return nil, err @@ -557,6 +564,7 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er // j.list.mu.Lock() // defer j.list.mu.Unlock() + return &withProvenance{CachedResult: res, j: j, e: e}, nil } diff --git a/solver/simple.go b/solver/simple.go index 51402e484..7d19f8eaf 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -16,28 +16,26 @@ import ( ) type simpleSolver struct { - resolveOpFunc ResolveOpFunc - solver *Solver - job *Job + resolveOpFunc ResolveOpFunc + solver *Solver + job *Job + flightControl *flightControl + resultCache *resultCache + cacheKeyManager *cacheKeyManager + mu sync.Mutex } -var cache = map[string]Result{} -var cacheMaps = map[string]*simpleCacheMap{} -var execInProgress = map[string]struct{}{} -var mu = sync.Mutex{} - -type cacheMapDep struct { - selector string - computed string -} - -type simpleCacheMap struct { - digest string - inputs []string - deps []cacheMapDep +func newSimpleSolver(resolveOpFunc ResolveOpFunc, solver *Solver) *simpleSolver { + return &simpleSolver{ + cacheKeyManager: newCacheKeyManager(), + resultCache: newResultCache(), + flightControl: newFlightControl(time.Millisecond * 100), + resolveOpFunc: resolveOpFunc, + solver: solver, + } } -func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) { +func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResult, error) { // Ordered list of vertices to build. digests, vertices := s.exploreVertices(e) @@ -50,20 +48,12 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return nil, errors.Errorf("digest %s not found", d) } - mu.Lock() - // TODO: replace busy-wait loop with a wait-for-channel-to-close approach - for { - if _, shouldWait := execInProgress[d.String()]; !shouldWait { - execInProgress[d.String()] = struct{}{} - mu.Unlock() - break - } - mu.Unlock() - time.Sleep(time.Millisecond * 10) - mu.Lock() - } + // Ensure we don't have multiple threads working on the same digest. + wait, done := s.flightControl.acquire(ctx, d.String()) + + <-wait - st := s.createState(vertex) + st := s.createState(vertex, job) op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) @@ -81,45 +71,38 @@ func (s *simpleSolver) build(ctx context.Context, e Edge) (CachedResult, error) return nil, err } - cacheKey, err := s.cacheKey(ctx, d.String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) if err != nil { return nil, err } - mu.Lock() - if v, ok := cache[cacheKey]; ok && v != nil { - delete(execInProgress, d.String()) - mu.Unlock() + if v, ok := s.resultCache.get(cacheKey); ok && v != nil { + done() ctx = progress.WithProgress(ctx, st.mpw) notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) notifyCompleted(nil, true) ret = v continue } - mu.Unlock() results, _, err := op.Exec(ctx, inputs) if err != nil { - mu.Lock() - delete(execInProgress, d.String()) - mu.Unlock() return nil, err } res := results[int(e.Index)] ret = res - mu.Lock() - delete(execInProgress, d.String()) - cache[cacheKey] = res - mu.Unlock() + s.resultCache.set(cacheKey, res) + + done() } return NewCachedResult(ret, []ExportableCacheKey{}), nil } // createState creates a new state struct with required and placeholder values. -func (s *simpleSolver) createState(vertex Vertex) *state { +func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { defaultCache := NewInMemoryCacheManager() st := &state{ @@ -140,14 +123,41 @@ func (s *simpleSolver) createState(vertex Vertex) *state { } st.jobs = map[*Job]struct{}{ - s.job: {}, + job: {}, } - st.mpw.Add(s.job.pw) + st.mpw.Add(job.pw) return st } +func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { + + digests := []digest.Digest{e.Vertex.Digest()} + vertices := map[digest.Digest]Vertex{ + e.Vertex.Digest(): e.Vertex, + } + + for _, edge := range e.Vertex.Inputs() { + d, v := s.exploreVertices(edge) + digests = append(d, digests...) + for key, value := range v { + vertices[key] = value + } + } + + ret := []digest.Digest{} + m := map[digest.Digest]struct{}{} + for _, d := range digests { + if _, ok := m[d]; !ok { + ret = append(ret, d) + m[d] = struct{}{} + } + } + + return ret, vertices +} + func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. @@ -162,15 +172,13 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V for i, in := range vertex.Inputs() { // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKey(ctx, in.Vertex.Digest().String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, in.Vertex.Digest().String()) if err != nil { return nil, err } // Lookup the result for that cache key. - mu.Lock() - res, ok := cache[cacheKey] - mu.Unlock() + res, ok := s.resultCache.get(cacheKey) if !ok { return nil, errors.Errorf("cache key not found: %s", cacheKey) } @@ -209,19 +217,45 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V inputs = append(inputs, res) } - mu.Lock() - cacheMaps[vertex.Digest().String()] = &scm - mu.Unlock() + s.cacheKeyManager.add(vertex.Digest().String(), &scm) return inputs, nil } +type cacheKeyManager struct { + cacheMaps map[string]*simpleCacheMap + mu sync.Mutex +} + +type cacheMapDep struct { + selector string + computed string +} + +type simpleCacheMap struct { + digest string + inputs []string + deps []cacheMapDep +} + +func newCacheKeyManager() *cacheKeyManager { + return &cacheKeyManager{ + cacheMaps: map[string]*simpleCacheMap{}, + } +} + +func (m *cacheKeyManager) add(key string, s *simpleCacheMap) { + m.mu.Lock() + m.cacheMaps[key] = s + m.mu.Unlock() +} + // cacheKey recursively generates a cache key based on a sequence of ancestor // operations & their cacheable values. -func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { +func (m *cacheKeyManager) cacheKey(ctx context.Context, d string) (string, error) { h := sha256.New() - err := s.cacheKeyRecurse(ctx, d, h) + err := m.cacheKeyRecurse(ctx, d, h) if err != nil { return "", err } @@ -229,16 +263,16 @@ func (s *simpleSolver) cacheKey(ctx context.Context, d string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -func (s *simpleSolver) cacheKeyRecurse(ctx context.Context, d string, h hash.Hash) error { - mu.Lock() - c, ok := cacheMaps[d] - mu.Unlock() +func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash.Hash) error { + m.mu.Lock() + c, ok := m.cacheMaps[d] + m.mu.Unlock() if !ok { return errors.New("missing cache map key") } for _, in := range c.inputs { - err := s.cacheKeyRecurse(ctx, in, h) + err := m.cacheKeyRecurse(ctx, in, h) if err != nil { return err } @@ -257,29 +291,78 @@ func (s *simpleSolver) cacheKeyRecurse(ctx context.Context, d string, h hash.Has return nil } -func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Digest]Vertex) { +type flightControl struct { + wait time.Duration + active map[string]struct{} + mu sync.Mutex +} - digests := []digest.Digest{e.Vertex.Digest()} - vertices := map[digest.Digest]Vertex{ - e.Vertex.Digest(): e.Vertex, - } +func newFlightControl(wait time.Duration) *flightControl { + return &flightControl{wait: wait, active: map[string]struct{}{}} +} - for _, edge := range e.Vertex.Inputs() { - d, v := s.exploreVertices(edge) - digests = append(d, digests...) - for key, value := range v { - vertices[key] = value - } +func (f *flightControl) acquire(ctx context.Context, d string) (<-chan struct{}, func()) { + + ch := make(chan struct{}) + + closer := func() { + f.mu.Lock() + delete(f.active, d) + f.mu.Unlock() } - ret := []digest.Digest{} - m := map[digest.Digest]struct{}{} - for _, d := range digests { - if _, ok := m[d]; !ok { - ret = append(ret, d) - m[d] = struct{}{} + go func() { + tick := time.NewTicker(f.wait) + defer tick.Stop() + // A function is used here as the above ticker does not execute + // immediately. + check := func() bool { + f.mu.Lock() + if _, ok := f.active[d]; !ok { + f.active[d] = struct{}{} + close(ch) + f.mu.Unlock() + return true + } + f.mu.Unlock() + return false } - } + if check() { + return + } + for { + select { + case <-tick.C: + if check() { + return + } + case <-ctx.Done(): + return + } + } + }() - return ret, vertices + return ch, closer +} + +type resultCache struct { + cache map[string]Result + mu sync.Mutex +} + +func newResultCache() *resultCache { + return &resultCache{cache: map[string]Result{}} +} + +func (c *resultCache) set(key string, r Result) { + c.mu.Lock() + c.cache[key] = r + c.mu.Unlock() +} + +func (c *resultCache) get(key string) (Result, bool) { + c.mu.Lock() + r, ok := c.cache[key] + c.mu.Unlock() + return r, ok } From d4f929ab1f4e96a4feeddcde020f7b0472afa910 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 22 Mar 2024 12:27:35 -0700 Subject: [PATCH 15/54] Move processing to new func --- solver/simple.go | 82 ++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 7d19f8eaf..5919eccf4 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -48,57 +48,63 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return nil, errors.Errorf("digest %s not found", d) } - // Ensure we don't have multiple threads working on the same digest. - wait, done := s.flightControl.acquire(ctx, d.String()) + res, err := s.buildOne(ctx, d, vertex, job, e) + if err != nil { + return nil, err + } - <-wait + ret = res + } - st := s.createState(vertex, job) + return NewCachedResult(ret, []ExportableCacheKey{}), nil +} - op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) +func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, error) { + // Ensure we don't have multiple threads working on the same digest. + wait, done := s.flightControl.acquire(ctx, d.String()) + defer done() + <-wait - // Required to access cache map results on state. - st.op = op + st := s.createState(vertex, job) - // CacheMap populates required fields in SourceOp. - cm, err := op.CacheMap(ctx, int(e.Index)) - if err != nil { - return nil, err - } - - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) - if err != nil { - return nil, err - } + op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) - if err != nil { - return nil, err - } + // Required to access cache map results on state. + st.op = op - if v, ok := s.resultCache.get(cacheKey); ok && v != nil { - done() - ctx = progress.WithProgress(ctx, st.mpw) - notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - notifyCompleted(nil, true) - ret = v - continue - } + // CacheMap populates required fields in SourceOp. + cm, err := op.CacheMap(ctx, int(e.Index)) + if err != nil { + return nil, err + } - results, _, err := op.Exec(ctx, inputs) - if err != nil { - return nil, err - } + inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) + if err != nil { + return nil, err + } - res := results[int(e.Index)] - ret = res + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) + if err != nil { + return nil, err + } - s.resultCache.set(cacheKey, res) + if v, ok := s.resultCache.get(cacheKey); ok && v != nil { + ctx = progress.WithProgress(ctx, st.mpw) + notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) + notifyCompleted(nil, true) + return v, nil + } - done() + results, _, err := op.Exec(ctx, inputs) + if err != nil { + return nil, err } - return NewCachedResult(ret, []ExportableCacheKey{}), nil + res := results[int(e.Index)] + + s.resultCache.set(cacheKey, res) + + return res, nil } // createState creates a new state struct with required and placeholder values. From a820067c8dd5f74d5d112c632aa1a94e9adfcbfc Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 25 Mar 2024 11:46:16 -0700 Subject: [PATCH 16/54] Rename flight control struct --- solver/simple.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 5919eccf4..ba5719505 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -19,7 +19,7 @@ type simpleSolver struct { resolveOpFunc ResolveOpFunc solver *Solver job *Job - flightControl *flightControl + parallelGuard *parallelGuard resultCache *resultCache cacheKeyManager *cacheKeyManager mu sync.Mutex @@ -29,7 +29,7 @@ func newSimpleSolver(resolveOpFunc ResolveOpFunc, solver *Solver) *simpleSolver return &simpleSolver{ cacheKeyManager: newCacheKeyManager(), resultCache: newResultCache(), - flightControl: newFlightControl(time.Millisecond * 100), + parallelGuard: newParallelGuard(time.Millisecond * 100), resolveOpFunc: resolveOpFunc, solver: solver, } @@ -61,7 +61,7 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, error) { // Ensure we don't have multiple threads working on the same digest. - wait, done := s.flightControl.acquire(ctx, d.String()) + wait, done := s.parallelGuard.acquire(ctx, d.String()) defer done() <-wait @@ -297,17 +297,17 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash. return nil } -type flightControl struct { +type parallelGuard struct { wait time.Duration active map[string]struct{} mu sync.Mutex } -func newFlightControl(wait time.Duration) *flightControl { - return &flightControl{wait: wait, active: map[string]struct{}{}} +func newParallelGuard(wait time.Duration) *parallelGuard { + return ¶llelGuard{wait: wait, active: map[string]struct{}{}} } -func (f *flightControl) acquire(ctx context.Context, d string) (<-chan struct{}, func()) { +func (f *parallelGuard) acquire(ctx context.Context, d string) (<-chan struct{}, func()) { ch := make(chan struct{}) From 24c41483877ebca66242e43255e6f84635d07a0e Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 14:02:49 -0700 Subject: [PATCH 17/54] Cache IDs & fix BK result caching --- solver/jobs.go | 12 ++- solver/llbsolver/provenance.go | 3 + solver/llbsolver/solver.go | 6 +- solver/simple.go | 144 ++++++++++++++++++++++++++++++--- worker/base/worker.go | 4 + worker/simple.go | 57 +++++++++++++ 6 files changed, 208 insertions(+), 18 deletions(-) create mode 100644 worker/simple.go diff --git a/solver/jobs.go b/solver/jobs.go index 665ebebb3..e39c593b1 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -256,8 +256,10 @@ type Job struct { } type SolverOpt struct { - ResolveOpFunc ResolveOpFunc - DefaultCache CacheManager + ResolveOpFunc ResolveOpFunc + DefaultCache CacheManager + WorkerResultGetter workerResultGetter + CommitRefFunc CommitRefFunc } func NewSolver(opts SolverOpt) *Solver { @@ -274,7 +276,11 @@ func NewSolver(opts SolverOpt) *Solver { // TODO: This should be hoisted up a few layers as not to be bound to the // original solver. For now, we just need a convenient place to initialize // it once. - simple := newSimpleSolver(opts.ResolveOpFunc, jl) + c, err := newDiskCache(opts.WorkerResultGetter) + if err != nil { + panic(err) // TODO: Handle error appropriately once the new solver code is moved. + } + simple := newSimpleSolver(opts.ResolveOpFunc, opts.CommitRefFunc, jl, c) jl.simple = simple jl.s = newScheduler(jl) diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index eca2ac14d..ae794ee02 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -269,6 +269,9 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc switch op := pp.(type) { case *ops.SourceOp: id, pin := op.Pin() + if pin == "" { // Hack: latest cache opt changes led to an empty value here. Investigate. + return nil + } err := id.Capture(c, pin) if err != nil { return err diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index ccc3504c0..e7528e8f5 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -119,8 +119,10 @@ func New(opt Opt) (*Solver, error) { s.sysSampler = sampler s.solver = solver.NewSolver(solver.SolverOpt{ - ResolveOpFunc: s.resolver(), - DefaultCache: opt.CacheManager, + ResolveOpFunc: s.resolver(), + DefaultCache: opt.CacheManager, + WorkerResultGetter: worker.NewWorkerResultGetter(opt.WorkerController), + CommitRefFunc: worker.FinalizeRef, }) return s, nil } diff --git a/solver/simple.go b/solver/simple.go index ba5719505..1d7cf8260 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -9,28 +9,35 @@ import ( "sync" "time" + "github.com/docker/docker/errdefs" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/tracing" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) +type CommitRefFunc func(ctx context.Context, result Result) error + type simpleSolver struct { resolveOpFunc ResolveOpFunc + commitRefFunc CommitRefFunc solver *Solver job *Job parallelGuard *parallelGuard - resultCache *resultCache + resultCache resultCache cacheKeyManager *cacheKeyManager mu sync.Mutex } -func newSimpleSolver(resolveOpFunc ResolveOpFunc, solver *Solver) *simpleSolver { +func newSimpleSolver(resolveOpFunc ResolveOpFunc, commitRefFunc CommitRefFunc, solver *Solver, cache resultCache) *simpleSolver { return &simpleSolver{ cacheKeyManager: newCacheKeyManager(), - resultCache: newResultCache(), + resultCache: cache, parallelGuard: newParallelGuard(time.Millisecond * 100), resolveOpFunc: resolveOpFunc, + commitRefFunc: commitRefFunc, solver: solver, } } @@ -72,6 +79,9 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver // Required to access cache map results on state. st.op = op + // Add cache opts to context as they will be accessed by cache retrieval. + ctx = withAncestorCacheOpts(ctx, st) + // CacheMap populates required fields in SourceOp. cm, err := op.CacheMap(ctx, int(e.Index)) if err != nil { @@ -88,7 +98,12 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } - if v, ok := s.resultCache.get(cacheKey); ok && v != nil { + v, ok, err := s.resultCache.get(ctx, cacheKey) + if err != nil { + return nil, err + } + + if ok && v != nil { ctx = progress.WithProgress(ctx, st.mpw) notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) notifyCompleted(nil, true) @@ -100,9 +115,21 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } + // Ensure all results are finalized (committed to cache). It may be better + // to background these calls at some point. + for _, res := range results { + err = s.commitRefFunc(ctx, res) + if err != nil { + return nil, err + } + } + res := results[int(e.Index)] - s.resultCache.set(cacheKey, res) + err = s.resultCache.set(ctx, cacheKey, res) + if err != nil { + return nil, err + } return res, nil } @@ -134,6 +161,14 @@ func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { st.mpw.Add(job.pw) + // Hack: this is used in combination with withAncestorCacheOpts to pass + // necessary dependency information to a few caching components. We'll need + // to expire these keys somehow. We should also move away from using the + // actives map, but it's still being used by withAncestorCacheOpts for now. + s.solver.mu.Lock() + s.solver.actives[vertex.Digest()] = st + s.solver.mu.Unlock() + return st } @@ -167,7 +202,6 @@ func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Dige func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. - // TODO: handle cm.Opts (CacheOpts)? scm := simpleCacheMap{ digest: cm.Digest.String(), deps: make([]cacheMapDep, len(cm.Deps)), @@ -184,7 +218,11 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V } // Lookup the result for that cache key. - res, ok := s.resultCache.get(cacheKey) + res, ok, err := s.resultCache.get(ctx, cacheKey) + if err != nil { + return nil, err + } + if !ok { return nil, errors.Errorf("cache key not found: %s", cacheKey) } @@ -351,24 +389,104 @@ func (f *parallelGuard) acquire(ctx context.Context, d string) (<-chan struct{}, return ch, closer } -type resultCache struct { +type resultCache interface { + set(ctx context.Context, key string, r Result) error + get(ctx context.Context, key string) (Result, bool, error) +} + +type inMemCache struct { cache map[string]Result mu sync.Mutex } -func newResultCache() *resultCache { - return &resultCache{cache: map[string]Result{}} +func newInMemCache() *inMemCache { + return &inMemCache{cache: map[string]Result{}} } -func (c *resultCache) set(key string, r Result) { +func (c *inMemCache) set(ctx context.Context, key string, r Result) error { c.mu.Lock() c.cache[key] = r c.mu.Unlock() + return nil } -func (c *resultCache) get(key string) (Result, bool) { +func (c *inMemCache) get(ctx context.Context, key string) (Result, bool, error) { c.mu.Lock() r, ok := c.cache[key] c.mu.Unlock() - return r, ok + return r, ok, nil +} + +var _ resultCache = &inMemCache{} + +type diskCache struct { + resultGetter workerResultGetter + db *bolt.DB + bucketName string } + +type workerResultGetter interface { + Get(ctx context.Context, id string) (Result, error) +} + +func newDiskCache(resultGetter workerResultGetter) (*diskCache, error) { + c := &diskCache{ + bucketName: "ids", + resultGetter: resultGetter, + } + err := c.init() + if err != nil { + return nil, err + } + return c, nil +} + +func (c *diskCache) init() error { + // TODO: pass in root config directory. + db, err := bolt.Open("/tmp/earthly/buildkit/simple.db", 0600, nil) + if err != nil { + return err + } + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("ids")) + return err + }) + if err != nil { + return err + } + c.db = db + return nil +} + +func (c *diskCache) set(ctx context.Context, key string, r Result) error { + return c.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(c.bucketName)) + return b.Put([]byte(key), []byte(r.ID())) + }) +} + +func (c *diskCache) get(ctx context.Context, key string) (Result, bool, error) { + var id string + err := c.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(c.bucketName)) + id = string(b.Get([]byte(key))) + return nil + }) + if err != nil { + return nil, false, err + } + if id == "" { + return nil, false, nil + } + res, err := c.resultGetter.Get(ctx, id) + if err != nil { + if errdefs.IsNotFound(err) { + bklog.G(ctx).Warnf("failed to get cached result from worker: %v", err) + return nil, false, nil + } + return nil, false, err + } + return res, true, nil +} + +var _ resultCache = &diskCache{} diff --git a/worker/base/worker.go b/worker/base/worker.go index e98f76214..d44521988 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -13,6 +13,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes/docker" + "github.com/davecgh/go-spew/spew" "github.com/docker/docker/pkg/idtools" "github.com/hashicorp/go-multierror" "github.com/moby/buildkit/cache" @@ -297,6 +298,8 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm ref, err := w.CacheMgr.Get(ctx, id, pg, opts...) var needsRemoteProviders cache.NeedsRemoteProviderError if errors.As(err, &needsRemoteProviders) { + fmt.Println("Trying again with cache opts") + if optGetter := solver.CacheOptGetterOf(ctx); optGetter != nil { var keys []interface{} for _, dgst := range needsRemoteProviders { @@ -310,6 +313,7 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm } } } + spew.Dump(descHandlers) opts = append(opts, descHandlers) ref, err = w.CacheMgr.Get(ctx, id, pg, opts...) } diff --git a/worker/simple.go b/worker/simple.go new file mode 100644 index 000000000..7565d7b74 --- /dev/null +++ b/worker/simple.go @@ -0,0 +1,57 @@ +package worker + +import ( + "context" + + "github.com/containerd/nydus-snapshotter/pkg/errdefs" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/solver" +) + +// WorkerResultGetter abstracts the work involved in loading a Result from a +// worker using a ref ID. +type WorkerResultGetter struct { + wc *Controller +} + +// NewWorkerResultGetter creates and returns a new *WorkerResultGetter. +func NewWorkerResultGetter(wc *Controller) *WorkerResultGetter { + return &WorkerResultGetter{wc: wc} +} + +// Get a cached results from a worker. +func (w *WorkerResultGetter) Get(ctx context.Context, id string) (solver.Result, error) { + workerID, refID, err := parseWorkerRef(id) + if err != nil { + return nil, err + } + + worker, err := w.wc.Get(workerID) + if err != nil { + return nil, err + } + + ref, err := worker.LoadRef(ctx, refID, false) + if err != nil { + if cache.IsNotFound(err) { + return nil, errdefs.ErrNotFound + } + return nil, err + } + + return NewWorkerRefResult(ref, worker), nil +} + +// FinalizeRef is a convenience function that calls Finalize on a Result's +// ImmutableRef. The 'worker' package cannot be imported by 'solver' due to an +// import cycle, so this function is passed in with solver.SolverOpt. +func FinalizeRef(ctx context.Context, res solver.Result) error { + sys := res.Sys() + if w, ok := sys.(*WorkerRef); ok { + err := w.ImmutableRef.Finalize(ctx) + if err != nil { + return err + } + } + return nil +} From 9b06a9798f96bea2e6bb58974dfd9ebd54f78eb8 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 14:05:05 -0700 Subject: [PATCH 18/54] Remove debug statements --- worker/base/worker.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/worker/base/worker.go b/worker/base/worker.go index d44521988..e98f76214 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -13,7 +13,6 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes/docker" - "github.com/davecgh/go-spew/spew" "github.com/docker/docker/pkg/idtools" "github.com/hashicorp/go-multierror" "github.com/moby/buildkit/cache" @@ -298,8 +297,6 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm ref, err := w.CacheMgr.Get(ctx, id, pg, opts...) var needsRemoteProviders cache.NeedsRemoteProviderError if errors.As(err, &needsRemoteProviders) { - fmt.Println("Trying again with cache opts") - if optGetter := solver.CacheOptGetterOf(ctx); optGetter != nil { var keys []interface{} for _, dgst := range needsRemoteProviders { @@ -313,7 +310,6 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm } } } - spew.Dump(descHandlers) opts = append(opts, descHandlers) ref, err = w.CacheMgr.Get(ctx, id, pg, opts...) } From dfe4db5d16a0ab3282bc77e067486649fd06491c Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 14:06:38 -0700 Subject: [PATCH 19/54] Added comment --- solver/simple.go | 1 + 1 file changed, 1 insertion(+) diff --git a/solver/simple.go b/solver/simple.go index 1d7cf8260..e85d5fd62 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -18,6 +18,7 @@ import ( bolt "go.etcd.io/bbolt" ) +// CommitRefFunc can be used to finalize a Result's ImmutableRef. type CommitRefFunc func(ctx context.Context, result Result) error type simpleSolver struct { From 0e34c18492033f95ff1939227cbd35a361bbc638 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 16:37:09 -0700 Subject: [PATCH 20/54] Fix error check --- solver/simple.go | 18 ++++++++++++++---- worker/simple.go | 3 +-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index e85d5fd62..aab6e12b0 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/docker/docker/errdefs" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/tracing" @@ -18,6 +17,8 @@ import ( bolt "go.etcd.io/bbolt" ) +var ErrRefNotFound = errors.New("ref not found") + // CommitRefFunc can be used to finalize a Result's ImmutableRef. type CommitRefFunc func(ctx context.Context, result Result) error @@ -444,7 +445,7 @@ func newDiskCache(resultGetter workerResultGetter) (*diskCache, error) { func (c *diskCache) init() error { // TODO: pass in root config directory. - db, err := bolt.Open("/tmp/earthly/buildkit/simple.db", 0600, nil) + db, err := bolt.Open("/tmp/earthly/buildkit/simple.db", 0755, nil) if err != nil { return err } @@ -481,8 +482,10 @@ func (c *diskCache) get(ctx context.Context, key string) (Result, bool, error) { } res, err := c.resultGetter.Get(ctx, id) if err != nil { - if errdefs.IsNotFound(err) { - bklog.G(ctx).Warnf("failed to get cached result from worker: %v", err) + if errors.Is(err, ErrRefNotFound) { + if err := c.delete(ctx, key); err != nil { + bklog.G(ctx).Warnf("failed to delete cache key: %v", err) + } return nil, false, nil } return nil, false, err @@ -490,4 +493,11 @@ func (c *diskCache) get(ctx context.Context, key string) (Result, bool, error) { return res, true, nil } +func (c *diskCache) delete(_ context.Context, key string) error { + return c.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(c.bucketName)) + return b.Delete([]byte(key)) + }) +} + var _ resultCache = &diskCache{} diff --git a/worker/simple.go b/worker/simple.go index 7565d7b74..c2ebb6fda 100644 --- a/worker/simple.go +++ b/worker/simple.go @@ -3,7 +3,6 @@ package worker import ( "context" - "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/solver" ) @@ -34,7 +33,7 @@ func (w *WorkerResultGetter) Get(ctx context.Context, id string) (solver.Result, ref, err := worker.LoadRef(ctx, refID, false) if err != nil { if cache.IsNotFound(err) { - return nil, errdefs.ErrNotFound + return nil, solver.ErrRefNotFound } return nil, err } From cf6f1e9371d2cd75aaca3a665f4a36fce2168432 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 17:30:01 -0700 Subject: [PATCH 21/54] Pass in root dir --- cmd/buildkitd/main.go | 1 + control/control.go | 2 ++ solver/jobs.go | 3 ++- solver/llbsolver/solver.go | 2 ++ solver/simple.go | 8 +++++--- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index fd316b9fc..0b0d61498 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -843,6 +843,7 @@ func newController(c *cli.Context, cfg *config.Config, shutdownCh chan struct{}) LeaseManager: w.LeaseManager(), ContentStore: w.ContentStore(), HistoryConfig: cfg.History, + RootDir: cfg.Root, }) } diff --git a/control/control.go b/control/control.go index a9816692d..7b7a158e8 100644 --- a/control/control.go +++ b/control/control.go @@ -68,6 +68,7 @@ type Opt struct { LeaseManager *leaseutil.Manager ContentStore *containerdsnapshot.Store HistoryConfig *config.HistoryConfig + RootDir string } type Controller struct { // TODO: ControlService @@ -105,6 +106,7 @@ func NewController(opt Opt) (*Controller, error) { SessionManager: opt.SessionManager, Entitlements: opt.Entitlements, HistoryQueue: hq, + RootDir: opt.RootDir, }) if err != nil { return nil, errors.Wrap(err, "failed to create solver") diff --git a/solver/jobs.go b/solver/jobs.go index e39c593b1..5116644d1 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -260,6 +260,7 @@ type SolverOpt struct { DefaultCache CacheManager WorkerResultGetter workerResultGetter CommitRefFunc CommitRefFunc + RootDir string } func NewSolver(opts SolverOpt) *Solver { @@ -276,7 +277,7 @@ func NewSolver(opts SolverOpt) *Solver { // TODO: This should be hoisted up a few layers as not to be bound to the // original solver. For now, we just need a convenient place to initialize // it once. - c, err := newDiskCache(opts.WorkerResultGetter) + c, err := newDiskCache(opts.WorkerResultGetter, opts.RootDir) if err != nil { panic(err) // TODO: Handle error appropriately once the new solver code is moved. } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index e7528e8f5..5cc90aa5f 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -79,6 +79,7 @@ type Opt struct { WorkerController *worker.Controller HistoryQueue *HistoryQueue ResourceMonitor *resources.Monitor + RootDir string } type Solver struct { @@ -123,6 +124,7 @@ func New(opt Opt) (*Solver, error) { DefaultCache: opt.CacheManager, WorkerResultGetter: worker.NewWorkerResultGetter(opt.WorkerController), CommitRefFunc: worker.FinalizeRef, + RootDir: opt.RootDir, }) return s, nil } diff --git a/solver/simple.go b/solver/simple.go index aab6e12b0..b68e9cd0f 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -6,6 +6,7 @@ import ( "fmt" "hash" "io" + "path/filepath" "sync" "time" @@ -425,16 +426,18 @@ type diskCache struct { resultGetter workerResultGetter db *bolt.DB bucketName string + rootDir string } type workerResultGetter interface { Get(ctx context.Context, id string) (Result, error) } -func newDiskCache(resultGetter workerResultGetter) (*diskCache, error) { +func newDiskCache(resultGetter workerResultGetter, rootDir string) (*diskCache, error) { c := &diskCache{ bucketName: "ids", resultGetter: resultGetter, + rootDir: rootDir, } err := c.init() if err != nil { @@ -444,8 +447,7 @@ func newDiskCache(resultGetter workerResultGetter) (*diskCache, error) { } func (c *diskCache) init() error { - // TODO: pass in root config directory. - db, err := bolt.Open("/tmp/earthly/buildkit/simple.db", 0755, nil) + db, err := bolt.Open(filepath.Join(c.rootDir, "ids.db"), 0755, nil) if err != nil { return err } From 0e52077c9c37900ec1a3b4e5a0599e5d954d8ec0 Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 3 Apr 2024 21:36:35 -0700 Subject: [PATCH 22/54] Don't wait on compute digest fail --- solver/simple.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index b68e9cd0f..363b34782 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -251,9 +251,10 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V if dep.ComputeDigestFunc != nil { compDigest, err := dep.ComputeDigestFunc(ctx, res, st) if err != nil { - return nil, err + bklog.G(ctx).Warnf("failed to compute digest: %v", err) + } else { + scm.deps[i].computed = compDigest.String() } - scm.deps[i].computed = compDigest.String() } // Add input references to the struct as to link dependencies. From 7f19ba4c02c97df99d9109cde6ff785b298b73cb Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 4 Apr 2024 12:58:53 -0700 Subject: [PATCH 23/54] Only pass opts to get --- solver/simple.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 363b34782..cf49dea37 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -83,7 +83,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver st.op = op // Add cache opts to context as they will be accessed by cache retrieval. - ctx = withAncestorCacheOpts(ctx, st) + cacheOptsCtx := withAncestorCacheOpts(ctx, st) // CacheMap populates required fields in SourceOp. cm, err := op.CacheMap(ctx, int(e.Index)) @@ -101,7 +101,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } - v, ok, err := s.resultCache.get(ctx, cacheKey) + v, ok, err := s.resultCache.get(cacheOptsCtx, cacheKey) if err != nil { return nil, err } From 1aee4b2a70cdc54ba262aca48143a4a6fa333a0c Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 4 Apr 2024 13:33:27 -0700 Subject: [PATCH 24/54] Revert --- solver/jobs.go | 2 ++ solver/simple.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 5116644d1..a156b7b6b 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/davecgh/go-spew/spew" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" @@ -954,6 +955,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, }() res, err := op.Exec(ctx, s.st, inputs) + spew.Dump(err) complete := true if err != nil { select { diff --git a/solver/simple.go b/solver/simple.go index cf49dea37..363b34782 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -83,7 +83,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver st.op = op // Add cache opts to context as they will be accessed by cache retrieval. - cacheOptsCtx := withAncestorCacheOpts(ctx, st) + ctx = withAncestorCacheOpts(ctx, st) // CacheMap populates required fields in SourceOp. cm, err := op.CacheMap(ctx, int(e.Index)) @@ -101,7 +101,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } - v, ok, err := s.resultCache.get(cacheOptsCtx, cacheKey) + v, ok, err := s.resultCache.get(ctx, cacheKey) if err != nil { return nil, err } From 69097fdd10a28c13d19d79c0b0e4032018f07ffd Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 4 Apr 2024 17:18:44 -0700 Subject: [PATCH 25/54] Fix not found display problem --- solver/jobs.go | 5 +++++ solver/simple.go | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index a156b7b6b..ad9889fa3 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -621,6 +621,11 @@ func (j *Job) CloseProgress() { } func (j *Job) Discard() error { + // TMP: Hack to prevent actives map deletes. + if true { + return nil + } + j.list.mu.Lock() defer j.list.mu.Unlock() diff --git a/solver/simple.go b/solver/simple.go index 363b34782..a93bf15dd 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -93,6 +93,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) if err != nil { + notifyError(ctx, st, false, err) return nil, err } @@ -107,9 +108,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver } if ok && v != nil { - ctx = progress.WithProgress(ctx, st.mpw) - notifyCompleted := notifyStarted(ctx, &st.clientVertex, true) - notifyCompleted(nil, true) + notifyError(ctx, st, true, nil) return v, nil } @@ -137,6 +136,12 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return res, nil } +func notifyError(ctx context.Context, st *state, cached bool, err error) { + ctx = progress.WithProgress(ctx, st.mpw) + notifyCompleted := notifyStarted(ctx, &st.clientVertex, cached) + notifyCompleted(err, cached) +} + // createState creates a new state struct with required and placeholder values. func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { defaultCache := NewInMemoryCacheManager() @@ -252,6 +257,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V compDigest, err := dep.ComputeDigestFunc(ctx, res, st) if err != nil { bklog.G(ctx).Warnf("failed to compute digest: %v", err) + return nil, err } else { scm.deps[i].computed = compDigest.String() } From 470d3fbecab55458e866e6cca19e160a8fdc92c3 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 5 Apr 2024 10:14:30 -0700 Subject: [PATCH 26/54] Remove debug --- solver/jobs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index ad9889fa3..9dfb79f1b 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/davecgh/go-spew/spew" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" @@ -960,7 +959,6 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, }() res, err := op.Exec(ctx, s.st, inputs) - spew.Dump(err) complete := true if err != nil { select { From faff83060c28bab86bc9d00d0f9c54f951606246 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 5 Apr 2024 14:31:22 -0700 Subject: [PATCH 27/54] Fixes --no-cache --- solver/simple.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index a93bf15dd..425f5c1a8 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -91,13 +91,22 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap) + // By default we generate a cache key that's not salted as the keys need to + // persist across builds. However, when cache is disabled, we scope the keys + // to the current session. This is because some jobs will be duplicated in a + // given build & will need to be cached in a limited way. + salt := "" + if op.IgnoreCache() { + salt = job.SessionID + } + + inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, salt) if err != nil { notifyError(ctx, st, false, err) return nil, err } - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, salt, d.String()) if err != nil { return nil, err } @@ -207,7 +216,7 @@ func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Dige return ret, vertices } -func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap) ([]Result, error) { +func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap, salt string) ([]Result, error) { // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. scm := simpleCacheMap{ @@ -220,7 +229,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V for i, in := range vertex.Inputs() { // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, in.Vertex.Digest().String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, salt, in.Vertex.Digest().String()) if err != nil { return nil, err } @@ -306,10 +315,12 @@ func (m *cacheKeyManager) add(key string, s *simpleCacheMap) { // cacheKey recursively generates a cache key based on a sequence of ancestor // operations & their cacheable values. -func (m *cacheKeyManager) cacheKey(ctx context.Context, d string) (string, error) { +func (m *cacheKeyManager) cacheKey(ctx context.Context, salt, digest string) (string, error) { h := sha256.New() - err := m.cacheKeyRecurse(ctx, d, h) + io.WriteString(h, salt) + + err := m.cacheKeyRecurse(ctx, digest, h) if err != nil { return "", err } From 636787bfcb14941ef3b1360e057cf64f9051abd0 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 8 Apr 2024 17:03:34 -0700 Subject: [PATCH 28/54] Fixes issue with ignore cache breaking cache key --- solver/simple.go | 40 ++++++++++++++++++++++------------------ worker/simple.go | 2 ++ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 425f5c1a8..3a01a5d55 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -27,7 +27,6 @@ type simpleSolver struct { resolveOpFunc ResolveOpFunc commitRefFunc CommitRefFunc solver *Solver - job *Job parallelGuard *parallelGuard resultCache resultCache cacheKeyManager *cacheKeyManager @@ -91,22 +90,13 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, err } - // By default we generate a cache key that's not salted as the keys need to - // persist across builds. However, when cache is disabled, we scope the keys - // to the current session. This is because some jobs will be duplicated in a - // given build & will need to be cached in a limited way. - salt := "" - if op.IgnoreCache() { - salt = job.SessionID - } - - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, salt) + inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, job) if err != nil { notifyError(ctx, st, false, err) return nil, err } - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, salt, d.String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) if err != nil { return nil, err } @@ -216,7 +206,7 @@ func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Dige return ret, vertices } -func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap, salt string) ([]Result, error) { +func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap, job *Job) ([]Result, error) { // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. scm := simpleCacheMap{ @@ -225,11 +215,22 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V inputs: make([]string, len(cm.Deps)), } + // By default we generate a cache key that's not salted as the keys need to + // persist across builds. However, when cache is disabled, we scope the keys + // to the current session. This is because some jobs will be duplicated in a + // given build & will need to be cached in a limited way. + if vertex.Options().IgnoreCache { + scm.salt = job.SessionID + } + var inputs []Result for i, in := range vertex.Inputs() { + + digest := in.Vertex.Digest().String() + // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, salt, in.Vertex.Digest().String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, digest) if err != nil { return nil, err } @@ -241,7 +242,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V } if !ok { - return nil, errors.Errorf("cache key not found: %s", cacheKey) + return nil, errors.Errorf("result not found for digest: %s", digest) } dep := cm.Deps[i] @@ -299,6 +300,7 @@ type simpleCacheMap struct { digest string inputs []string deps []cacheMapDep + salt string } func newCacheKeyManager() *cacheKeyManager { @@ -315,11 +317,9 @@ func (m *cacheKeyManager) add(key string, s *simpleCacheMap) { // cacheKey recursively generates a cache key based on a sequence of ancestor // operations & their cacheable values. -func (m *cacheKeyManager) cacheKey(ctx context.Context, salt, digest string) (string, error) { +func (m *cacheKeyManager) cacheKey(ctx context.Context, digest string) (string, error) { h := sha256.New() - io.WriteString(h, salt) - err := m.cacheKeyRecurse(ctx, digest, h) if err != nil { return "", err @@ -336,6 +336,10 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash. return errors.New("missing cache map key") } + if c.salt != "" { + io.WriteString(h, c.salt) + } + for _, in := range c.inputs { err := m.cacheKeyRecurse(ctx, in, h) if err != nil { diff --git a/worker/simple.go b/worker/simple.go index c2ebb6fda..dec48e1d3 100644 --- a/worker/simple.go +++ b/worker/simple.go @@ -5,6 +5,7 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" ) // WorkerResultGetter abstracts the work involved in loading a Result from a @@ -33,6 +34,7 @@ func (w *WorkerResultGetter) Get(ctx context.Context, id string) (solver.Result, ref, err := worker.LoadRef(ctx, refID, false) if err != nil { if cache.IsNotFound(err) { + bklog.G(ctx).Warnf("could not load ref from worker: %v", err) return nil, solver.ErrRefNotFound } return nil, err From 7019a4e2bca0391d7091013430ab145f06b7957e Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 10 Apr 2024 17:45:00 -0700 Subject: [PATCH 29/54] Enable job discard, exporter stub --- solver/jobs.go | 4 ---- solver/simple.go | 40 ++++++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 9dfb79f1b..8c543bcfc 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -620,10 +620,6 @@ func (j *Job) CloseProgress() { } func (j *Job) Discard() error { - // TMP: Hack to prevent actives map deletes. - if true { - return nil - } j.list.mu.Lock() defer j.list.mu.Unlock() diff --git a/solver/simple.go b/solver/simple.go index 3a01a5d55..ddc7e2f6c 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -49,7 +49,7 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul // Ordered list of vertices to build. digests, vertices := s.exploreVertices(e) - var ret Result + var ret CachedResult for _, d := range digests { vertex, ok := vertices[d] @@ -57,18 +57,18 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return nil, errors.Errorf("digest %s not found", d) } - res, err := s.buildOne(ctx, d, vertex, job, e) + res, expCacheKeys, err := s.buildOne(ctx, d, vertex, job, e) if err != nil { return nil, err } - ret = res + ret = NewCachedResult(res, expCacheKeys) } - return NewCachedResult(ret, []ExportableCacheKey{}), nil + return ret, nil } -func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, error) { +func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, []ExportableCacheKey, error) { // Ensure we don't have multiple threads working on the same digest. wait, done := s.parallelGuard.acquire(ctx, d.String()) defer done() @@ -87,33 +87,37 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver // CacheMap populates required fields in SourceOp. cm, err := op.CacheMap(ctx, int(e.Index)) if err != nil { - return nil, err + return nil, nil, err } inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, job) if err != nil { notifyError(ctx, st, false, err) - return nil, err + return nil, nil, err } cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) if err != nil { - return nil, err + return nil, nil, err } v, ok, err := s.resultCache.get(ctx, cacheKey) if err != nil { - return nil, err + return nil, nil, err + } + + expCacheKeys := []ExportableCacheKey{ + {Exporter: &simpleExporter{cacheKey: cacheKey}}, } if ok && v != nil { notifyError(ctx, st, true, nil) - return v, nil + return v, expCacheKeys, nil } results, _, err := op.Exec(ctx, inputs) if err != nil { - return nil, err + return nil, nil, err } // Ensure all results are finalized (committed to cache). It may be better @@ -121,7 +125,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver for _, res := range results { err = s.commitRefFunc(ctx, res) if err != nil { - return nil, err + return nil, nil, err } } @@ -129,10 +133,10 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver err = s.resultCache.set(ctx, cacheKey, res) if err != nil { - return nil, err + return nil, nil, err } - return res, nil + return res, expCacheKeys, nil } func notifyError(ctx context.Context, st *state, cached bool, err error) { @@ -525,3 +529,11 @@ func (c *diskCache) delete(_ context.Context, key string) error { } var _ resultCache = &diskCache{} + +type simpleExporter struct { + cacheKey string +} + +func (s *simpleExporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt CacheExportOpt) ([]CacheExporterRecord, error) { + return nil, nil +} From a1f27c587997785766cc0af7a1dd35665b0558fa Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 11 Apr 2024 12:01:48 -0700 Subject: [PATCH 30/54] Don't discard --- solver/jobs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solver/jobs.go b/solver/jobs.go index 8c543bcfc..f299c3df0 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -620,6 +620,9 @@ func (j *Job) CloseProgress() { } func (j *Job) Discard() error { + if true { + return nil + } j.list.mu.Lock() defer j.list.mu.Unlock() From 0bba91695422945dacca641a901dd507b3050b9d Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 11 Apr 2024 13:25:09 -0700 Subject: [PATCH 31/54] Add back state use --- solver/jobs.go | 4 ---- solver/simple.go | 27 +++++++++++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index f299c3df0..5116644d1 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -620,10 +620,6 @@ func (j *Job) CloseProgress() { } func (j *Job) Discard() error { - if true { - return nil - } - j.list.mu.Lock() defer j.list.mu.Unlock() diff --git a/solver/simple.go b/solver/simple.go index ddc7e2f6c..dbae49a3d 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -74,18 +74,13 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver defer done() <-wait - st := s.createState(vertex, job) - - op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) - - // Required to access cache map results on state. - st.op = op + st := s.state(vertex, job) // Add cache opts to context as they will be accessed by cache retrieval. ctx = withAncestorCacheOpts(ctx, st) // CacheMap populates required fields in SourceOp. - cm, err := op.CacheMap(ctx, int(e.Index)) + cm, err := st.op.CacheMap(ctx, int(e.Index)) if err != nil { return nil, nil, err } @@ -115,7 +110,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return v, expCacheKeys, nil } - results, _, err := op.Exec(ctx, inputs) + results, _, err := st.op.Exec(ctx, inputs) if err != nil { return nil, nil, err } @@ -145,6 +140,17 @@ func notifyError(ctx context.Context, st *state, cached bool, err error) { notifyCompleted(err, cached) } +func (s *simpleSolver) state(vertex Vertex, job *Job) *state { + s.solver.mu.RLock() + if st, ok := s.solver.actives[vertex.Digest()]; ok { + st.jobs[job] = struct{}{} + s.solver.mu.RUnlock() + return st + } + s.solver.mu.RUnlock() + return s.createState(vertex, job) +} + // createState creates a new state struct with required and placeholder values. func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { defaultCache := NewInMemoryCacheManager() @@ -180,6 +186,11 @@ func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { s.solver.actives[vertex.Digest()] = st s.solver.mu.Unlock() + op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) + + // Required to access cache map results on state. + st.op = op + return st } From 7f982923effb8c168b6ab93c6af61d6d86e88e0e Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 12 Apr 2024 12:10:32 -0700 Subject: [PATCH 32/54] Add log for context --- frontend/gateway/gateway.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index afdabeb0e..dbaea3c3d 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -842,7 +842,11 @@ func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileReq return &pb.ReadFileResponse{Data: dt}, nil } -func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (*pb.ReadDirResponse, error) { +func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (res *pb.ReadDirResponse, retErr error) { + defer func() { + bklog.G(ctx).Warnf("Error from ReadDir: %v", retErr) + }() + ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) ref, err := lbf.getImmutableRef(ctx, req.Ref, req.DirPath) From da937190d9e2c3b1b77df769267929f6bcc91452 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 12 Apr 2024 12:56:03 -0700 Subject: [PATCH 33/54] More logs --- frontend/gateway/gateway.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index dbaea3c3d..ed783da49 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -851,6 +851,7 @@ func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirReque ref, err := lbf.getImmutableRef(ctx, req.Ref, req.DirPath) if err != nil { + bklog.G(ctx).Warnf("Error from getImmutableRef: %v", err) return nil, err } @@ -867,6 +868,7 @@ func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirReque } entries, err := cacheutil.ReadDir(ctx, m, newReq) if err != nil { + bklog.G(ctx).Warnf("Error from cacheutil.ReadDir: %v", err) return nil, lbf.wrapSolveError(err) } From 6bec2550a99a2f1bfc392d0ab6f362d5f07905bf Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 12 Apr 2024 15:36:15 -0700 Subject: [PATCH 34/54] Add more logging --- solver/jobs.go | 2 ++ solver/llbsolver/ops/exec.go | 4 ++-- source/local/source.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 5116644d1..3ad943aef 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -10,6 +10,7 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/progress/controller" @@ -920,6 +921,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, defer func() { err = errdefs.WithOp(err, s.st.vtx.Sys()) err = errdefs.WrapVertex(err, s.st.origDigest) + bklog.G(ctx).Warnf("sharedOp#Exec error: %T %v", err, err) }() op, err := s.getOp() if err != nil { diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 1751f78f7..89b26f237 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -608,7 +608,7 @@ func (e *ExecOp) sendLocally(ctx context.Context, root executor.Mount, mounts [] } err = localhost.LocalhostPut(ctx, caller, finalSrc, finalDst) if err != nil { - return errors.Wrap(err, "error calling LocalhostExec") + return errors.Wrap(err, "error calling LocalhostExec 1") } } return nil @@ -625,7 +625,7 @@ func (e *ExecOp) execLocally(ctx context.Context, root executor.Mount, g session return e.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { err := localhost.LocalhostExec(ctx, caller, args, cwd, stdout, stderr) if err != nil { - return errors.Wrap(err, "error calling LocalhostExec") + return errors.Wrap(err, "error calling LocalhostExec 2") } return nil }) diff --git a/source/local/source.go b/source/local/source.go index 0fd41c03a..074fe1bcd 100644 --- a/source/local/source.go +++ b/source/local/source.go @@ -127,7 +127,7 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, ind // Hack: The encoded session ID here is breaking the simplified caching // approach in "simple.go". However, a consistent value here is likely // unreliable with multiple users. Figure out another option. - sessionID = "session-id" + sessionID = ls.src.SharedKeyHint dt, err := json.Marshal(struct { SessionID string From f8401c30590f4f9fe94a886a03d27ef567fe4392 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 12 Apr 2024 17:09:08 -0700 Subject: [PATCH 35/54] Update comment --- source/local/source.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/local/source.go b/source/local/source.go index 074fe1bcd..67665a00b 100644 --- a/source/local/source.go +++ b/source/local/source.go @@ -125,8 +125,9 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, ind } // Hack: The encoded session ID here is breaking the simplified caching - // approach in "simple.go". However, a consistent value here is likely - // unreliable with multiple users. Figure out another option. + // approach in "simple.go" as it differs for each request. Use the + // SharedKeyHint property which is provided by Earthly and is based off of + // the path & inode names. sessionID = ls.src.SharedKeyHint dt, err := json.Marshal(struct { From 31ad06f37ef7dd2afb19b01d7f4b2353995c87c3 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 12 Apr 2024 21:04:21 -0700 Subject: [PATCH 36/54] More logging --- solver/jobs.go | 1 + solver/llbsolver/ops/exec.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/solver/jobs.go b/solver/jobs.go index 3ad943aef..913606c12 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -956,6 +956,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, }() res, err := op.Exec(ctx, s.st, inputs) + bklog.G(ctx).Warnf("op#Exec error: %T %v", err, err) complete := true if err != nil { select { diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 89b26f237..702ab1c5d 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -28,6 +28,7 @@ import ( "github.com/moby/buildkit/solver/llbsolver/mounts" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress/logs" "github.com/moby/buildkit/util/semutil" utilsystem "github.com/moby/buildkit/util/system" @@ -435,6 +436,7 @@ func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu Stderr: stderr, StatsStream: statsStream, // earthly-specific }, nil) + bklog.G(ctx).Warnf("error from e.exec.Run %T %v", execErr, execErr) } for i, out := range p.OutputRefs { From eeba41c6d8b20b7d151819b58a50314c3487a6b0 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 14 Apr 2024 09:14:19 -0700 Subject: [PATCH 37/54] Output more errors --- solver/llbsolver/ops/exec.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 702ab1c5d..9dc178961 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -287,6 +287,10 @@ func addDefaultEnvvar(env []string, k, v string) []string { func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (results []solver.Result, err error) { trace.SpanFromContext(ctx).AddEvent("ExecOp started") + defer func() { + bklog.G(ctx).Warnf("Error from ExecOp#Exec %T %v", err, err) + }() + refs := make([]*worker.WorkerRef, len(inputs)) for i, inp := range inputs { var ok bool From 81a613ebd48b6ad7a34131712123f9b1ee0e8649 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 14 Apr 2024 09:15:10 -0700 Subject: [PATCH 38/54] Remove cancel --- solver/jobs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solver/jobs.go b/solver/jobs.go index 913606c12..b4ae7afbb 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -955,7 +955,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, notifyCompleted(retErr, false) }() - res, err := op.Exec(ctx, s.st, inputs) + res, err := op.Exec(context.WithoutCancel(ctx), s.st, inputs) bklog.G(ctx).Warnf("op#Exec error: %T %v", err, err) complete := true if err != nil { From b2064ffd4e4920036a4de818699a74c955e835f9 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 14 Apr 2024 10:21:39 -0700 Subject: [PATCH 39/54] Disable stats collection tmp --- executor/runcexecutor/executor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/executor/runcexecutor/executor.go b/executor/runcexecutor/executor.go index 40e0aa658..ddcfa3923 100644 --- a/executor/runcexecutor/executor.go +++ b/executor/runcexecutor/executor.go @@ -330,9 +330,9 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, if started != nil { close(started) } - if process.StatsStream != nil { - go w.monitorContainerStats(ctx, id, w.sampleFrequency, process.StatsStream) // earthly-specific - } + // if process.StatsStream != nil { + // go w.monitorContainerStats(ctx, id, w.sampleFrequency, process.StatsStream) // earthly-specific + // } if rec != nil { rec.Start() } From 221d8234760418f2e35c3b7781121756611d7f4e Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 15 Apr 2024 09:53:58 -0700 Subject: [PATCH 40/54] Remove WithCancel --- executor/runcexecutor/executor.go | 1 + solver/jobs.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/executor/runcexecutor/executor.go b/executor/runcexecutor/executor.go index ddcfa3923..7e349adb9 100644 --- a/executor/runcexecutor/executor.go +++ b/executor/runcexecutor/executor.go @@ -330,6 +330,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, if started != nil { close(started) } + // This may be cause of cancel // if process.StatsStream != nil { // go w.monitorContainerStats(ctx, id, w.sampleFrequency, process.StatsStream) // earthly-specific // } diff --git a/solver/jobs.go b/solver/jobs.go index b4ae7afbb..913606c12 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -955,7 +955,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, notifyCompleted(retErr, false) }() - res, err := op.Exec(context.WithoutCancel(ctx), s.st, inputs) + res, err := op.Exec(ctx, s.st, inputs) bklog.G(ctx).Warnf("op#Exec error: %T %v", err, err) complete := true if err != nil { From ecbd642adb8c6b2c596165ac4d0147b268bd84d9 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 15 Apr 2024 11:02:01 -0700 Subject: [PATCH 41/54] Cancel stats collector when done --- executor/runcexecutor/executor.go | 10 +++--- executor/runcexecutor/monitor_stats.go | 50 ++++++++++++++------------ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/executor/runcexecutor/executor.go b/executor/runcexecutor/executor.go index 7e349adb9..4c43f8646 100644 --- a/executor/runcexecutor/executor.go +++ b/executor/runcexecutor/executor.go @@ -323,6 +323,8 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, } } + statsCtx, statsCancel := context.WithCancel(ctx) + trace.SpanFromContext(ctx).AddEvent("Container created") err = w.run(ctx, id, bundle, process, func() { startedOnce.Do(func() { @@ -330,10 +332,9 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, if started != nil { close(started) } - // This may be cause of cancel - // if process.StatsStream != nil { - // go w.monitorContainerStats(ctx, id, w.sampleFrequency, process.StatsStream) // earthly-specific - // } + if process.StatsStream != nil { + go w.monitorContainerStats(statsCtx, id, w.sampleFrequency, process.StatsStream) // earthly-specific + } if rec != nil { rec.Start() } @@ -341,6 +342,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, }, true) releaseContainer := func(ctx context.Context) error { + statsCancel() err := w.runc.Delete(ctx, id, &runc.DeleteOpts{}) err1 := namespace.Close() if err == nil { diff --git a/executor/runcexecutor/monitor_stats.go b/executor/runcexecutor/monitor_stats.go index 79b63d872..4d80cf68d 100644 --- a/executor/runcexecutor/monitor_stats.go +++ b/executor/runcexecutor/monitor_stats.go @@ -46,33 +46,39 @@ func writeStatsToStream(w io.Writer, stats *runc.Stats) error { func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sampleFrequency time.Duration, statsWriter io.WriteCloser) { numFailuresAllowed := 10 - for { - // sleep at the top of the loop to give it time to start - time.Sleep(sampleFrequency) - stats, err := w.runc.Stats(ctx, id) - if err != nil { - if errors.Is(err, context.Canceled) { + timer := time.NewTimer(sampleFrequency) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + bklog.G(ctx).Infof("stats collection context done: %v", ctx.Err()) + case <-timer.C: + stats, err := w.runc.Stats(ctx, id) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + if numFailuresAllowed > 0 { + // allow the initial calls to runc.Stats to fail, for cases where the program didn't start within the initial + // sampleFrequency; this should only occur under heavy workloads + bklog.G(ctx).Warnf("ignoring runc stats collection error: %s", err) + numFailuresAllowed-- + continue + } + bklog.G(ctx).Errorf("runc stats collection error: %s", err) return } - if numFailuresAllowed > 0 { - // allow the initial calls to runc.Stats to fail, for cases where the program didn't start within the initial - // sampleFrequency; this should only occur under heavy workloads - bklog.G(ctx).Warnf("ignoring runc stats collection error: %s", err) - numFailuresAllowed-- - continue - } - bklog.G(ctx).Errorf("runc stats collection error: %s", err) - return - } - // once runc.Stats has succeeded, don't ignore future errors - numFailuresAllowed = 0 + // once runc.Stats has succeeded, don't ignore future errors + numFailuresAllowed = 0 - err = writeStatsToStream(statsWriter, stats) - if err != nil { - bklog.G(ctx).Errorf("failed to send runc stats to client-stream: %s", err) - return + err = writeStatsToStream(statsWriter, stats) + if err != nil { + bklog.G(ctx).Errorf("failed to send runc stats to client-stream: %s", err) + return + } } } } From 1839ae49ac95e0edff225219de8d036a76e418ea Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 15 Apr 2024 11:20:24 -0700 Subject: [PATCH 42/54] Add missing return --- executor/runcexecutor/monitor_stats.go | 1 + 1 file changed, 1 insertion(+) diff --git a/executor/runcexecutor/monitor_stats.go b/executor/runcexecutor/monitor_stats.go index 4d80cf68d..ddf184b2a 100644 --- a/executor/runcexecutor/monitor_stats.go +++ b/executor/runcexecutor/monitor_stats.go @@ -54,6 +54,7 @@ func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sam select { case <-ctx.Done(): bklog.G(ctx).Infof("stats collection context done: %v", ctx.Err()) + return case <-timer.C: stats, err := w.runc.Stats(ctx, id) if err != nil { From c92f5521cbf3240c32d1f4c1c0019e460a22be0a Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 15 Apr 2024 13:22:14 -0700 Subject: [PATCH 43/54] Format comments --- executor/runcexecutor/monitor_stats.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/executor/runcexecutor/monitor_stats.go b/executor/runcexecutor/monitor_stats.go index ddf184b2a..931ae041d 100644 --- a/executor/runcexecutor/monitor_stats.go +++ b/executor/runcexecutor/monitor_stats.go @@ -55,15 +55,17 @@ func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sam case <-ctx.Done(): bklog.G(ctx).Infof("stats collection context done: %v", ctx.Err()) return - case <-timer.C: + case <-timer.C: // Initial sleep will give container the chance to start. stats, err := w.runc.Stats(ctx, id) if err != nil { if errors.Is(err, context.Canceled) { return } if numFailuresAllowed > 0 { - // allow the initial calls to runc.Stats to fail, for cases where the program didn't start within the initial - // sampleFrequency; this should only occur under heavy workloads + // Allow the initial calls to runc.Stats to fail, for cases + // where the program didn't start within the initial + // sampleFrequency; this should only occur under heavy + // workloads. bklog.G(ctx).Warnf("ignoring runc stats collection error: %s", err) numFailuresAllowed-- continue @@ -72,7 +74,7 @@ func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sam return } - // once runc.Stats has succeeded, don't ignore future errors + // Once runc.Stats has succeeded, don't ignore future errors. numFailuresAllowed = 0 err = writeStatsToStream(statsWriter, stats) From 196c66b63b4cc050133cd8a4fc66a8fca131b331 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 15 Apr 2024 17:17:59 -0700 Subject: [PATCH 44/54] Remove debug messages --- frontend/gateway/gateway.go | 6 ------ solver/jobs.go | 3 --- solver/llbsolver/ops/exec.go | 4 ++-- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index ed783da49..0a1430664 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -843,15 +843,10 @@ func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileReq } func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (res *pb.ReadDirResponse, retErr error) { - defer func() { - bklog.G(ctx).Warnf("Error from ReadDir: %v", retErr) - }() - ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) ref, err := lbf.getImmutableRef(ctx, req.Ref, req.DirPath) if err != nil { - bklog.G(ctx).Warnf("Error from getImmutableRef: %v", err) return nil, err } @@ -868,7 +863,6 @@ func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirReque } entries, err := cacheutil.ReadDir(ctx, m, newReq) if err != nil { - bklog.G(ctx).Warnf("Error from cacheutil.ReadDir: %v", err) return nil, lbf.wrapSolveError(err) } diff --git a/solver/jobs.go b/solver/jobs.go index 913606c12..5116644d1 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -10,7 +10,6 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver/errdefs" - "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/progress/controller" @@ -921,7 +920,6 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, defer func() { err = errdefs.WithOp(err, s.st.vtx.Sys()) err = errdefs.WrapVertex(err, s.st.origDigest) - bklog.G(ctx).Warnf("sharedOp#Exec error: %T %v", err, err) }() op, err := s.getOp() if err != nil { @@ -956,7 +954,6 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, }() res, err := op.Exec(ctx, s.st, inputs) - bklog.G(ctx).Warnf("op#Exec error: %T %v", err, err) complete := true if err != nil { select { diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 9dc178961..8261ba8ea 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -614,7 +614,7 @@ func (e *ExecOp) sendLocally(ctx context.Context, root executor.Mount, mounts [] } err = localhost.LocalhostPut(ctx, caller, finalSrc, finalDst) if err != nil { - return errors.Wrap(err, "error calling LocalhostExec 1") + return errors.Wrap(err, "error calling LocalhostExec") } } return nil @@ -631,7 +631,7 @@ func (e *ExecOp) execLocally(ctx context.Context, root executor.Mount, g session return e.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { err := localhost.LocalhostExec(ctx, caller, args, cwd, stdout, stderr) if err != nil { - return errors.Wrap(err, "error calling LocalhostExec 2") + return errors.Wrap(err, "error calling LocalhostExec") } return nil }) From 76e4ef7434472b93a4b09ae7774b864ea3125672 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 16 Apr 2024 10:45:12 -0700 Subject: [PATCH 45/54] Remove logging --- frontend/gateway/gateway.go | 2 +- solver/llbsolver/ops/exec.go | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index 0a1430664..afdabeb0e 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -842,7 +842,7 @@ func (lbf *llbBridgeForwarder) ReadFile(ctx context.Context, req *pb.ReadFileReq return &pb.ReadFileResponse{Data: dt}, nil } -func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (res *pb.ReadDirResponse, retErr error) { +func (lbf *llbBridgeForwarder) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (*pb.ReadDirResponse, error) { ctx = tracing.ContextWithSpanFromContext(ctx, lbf.callCtx) ref, err := lbf.getImmutableRef(ctx, req.Ref, req.DirPath) diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 8261ba8ea..1751f78f7 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -28,7 +28,6 @@ import ( "github.com/moby/buildkit/solver/llbsolver/mounts" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress/logs" "github.com/moby/buildkit/util/semutil" utilsystem "github.com/moby/buildkit/util/system" @@ -287,10 +286,6 @@ func addDefaultEnvvar(env []string, k, v string) []string { func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (results []solver.Result, err error) { trace.SpanFromContext(ctx).AddEvent("ExecOp started") - defer func() { - bklog.G(ctx).Warnf("Error from ExecOp#Exec %T %v", err, err) - }() - refs := make([]*worker.WorkerRef, len(inputs)) for i, inp := range inputs { var ok bool @@ -440,7 +435,6 @@ func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu Stderr: stderr, StatsStream: statsStream, // earthly-specific }, nil) - bklog.G(ctx).Warnf("error from e.exec.Run %T %v", execErr, execErr) } for i, out := range p.OutputRefs { From ffa1bc7f3604f2a45e4ee02edc7cee5fb9280d4b Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 16 Apr 2024 14:24:59 -0700 Subject: [PATCH 46/54] More locking --- solver/simple.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index dbae49a3d..4ecb3e369 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -141,13 +141,12 @@ func notifyError(ctx context.Context, st *state, cached bool, err error) { } func (s *simpleSolver) state(vertex Vertex, job *Job) *state { - s.solver.mu.RLock() + s.solver.mu.Lock() + defer s.solver.mu.Unlock() if st, ok := s.solver.actives[vertex.Digest()]; ok { st.jobs[job] = struct{}{} - s.solver.mu.RUnlock() return st } - s.solver.mu.RUnlock() return s.createState(vertex, job) } @@ -182,9 +181,7 @@ func (s *simpleSolver) createState(vertex Vertex, job *Job) *state { // necessary dependency information to a few caching components. We'll need // to expire these keys somehow. We should also move away from using the // actives map, but it's still being used by withAncestorCacheOpts for now. - s.solver.mu.Lock() s.solver.actives[vertex.Digest()] = st - s.solver.mu.Unlock() op := newSharedOp(st.opts.ResolveOpFunc, st.opts.DefaultCache, st) From 850d6e6e7587b99fd6d765223dcd47d37851e041 Mon Sep 17 00:00:00 2001 From: Alex Couture-Beil Date: Fri, 19 Apr 2024 14:02:45 -0700 Subject: [PATCH 47/54] Indicate if token is an anonymous token --- session/auth/auth.pb.go | 119 +++++++++++++++------- session/auth/auth.proto | 56 +++++----- session/auth/authprovider/authprovider.go | 12 ++- 3 files changed, 118 insertions(+), 69 deletions(-) diff --git a/session/auth/auth.pb.go b/session/auth/auth.pb.go index e23a07fc8..d85b06a46 100644 --- a/session/auth/auth.pb.go +++ b/session/auth/auth.pb.go @@ -202,6 +202,7 @@ type FetchTokenResponse struct { Token string `protobuf:"bytes,1,opt,name=Token,proto3" json:"Token,omitempty"` ExpiresIn int64 `protobuf:"varint,2,opt,name=ExpiresIn,proto3" json:"ExpiresIn,omitempty"` IssuedAt int64 `protobuf:"varint,3,opt,name=IssuedAt,proto3" json:"IssuedAt,omitempty"` + Anonymous bool `protobuf:"varint,99,opt,name=Anonymous,proto3" json:"Anonymous,omitempty"` } func (m *FetchTokenResponse) Reset() { *m = FetchTokenResponse{} } @@ -257,6 +258,13 @@ func (m *FetchTokenResponse) GetIssuedAt() int64 { return 0 } +func (m *FetchTokenResponse) GetAnonymous() bool { + if m != nil { + return m.Anonymous + } + return false +} + type GetTokenAuthorityRequest struct { Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"` Salt []byte `protobuf:"bytes,2,opt,name=Salt,proto3" json:"Salt,omitempty"` @@ -467,40 +475,41 @@ func init() { func init() { proto.RegisterFile("auth.proto", fileDescriptor_8bbd6f3875b0e874) } var fileDescriptor_8bbd6f3875b0e874 = []byte{ - // 513 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xc7, 0xbd, 0x75, 0xd2, 0x36, 0x43, 0x0f, 0x74, 0x89, 0x90, 0x31, 0xd1, 0xaa, 0x32, 0x45, - 0xaa, 0x40, 0x58, 0x02, 0x24, 0x24, 0xb8, 0xb5, 0xe5, 0x2b, 0xe2, 0x52, 0x39, 0x7c, 0x48, 0xbd, - 0x20, 0xc7, 0x9e, 0x12, 0x0b, 0xc7, 0x0e, 0xde, 0x75, 0x85, 0x6f, 0xdc, 0xb9, 0xf0, 0x08, 0x1c, - 0x79, 0x14, 0x8e, 0x39, 0xf6, 0x48, 0x9c, 0x0b, 0xc7, 0x3c, 0x02, 0xf2, 0x66, 0x9d, 0x04, 0x1c, - 0xd2, 0xdc, 0xfc, 0x1f, 0xff, 0x77, 0xe6, 0xb7, 0x33, 0xa3, 0x05, 0x70, 0x53, 0xd1, 0xb3, 0x07, - 0x49, 0x2c, 0x62, 0x7a, 0xb5, 0x1f, 0x77, 0x33, 0xfb, 0x2c, 0x08, 0x91, 0x67, 0x91, 0x67, 0x9f, - 0xdf, 0xb7, 0x0e, 0x80, 0x1e, 0x27, 0xe8, 0x63, 0x24, 0x02, 0x37, 0xe4, 0x0e, 0x7e, 0x4a, 0x91, - 0x0b, 0x4a, 0xa1, 0xf6, 0x32, 0xe6, 0xc2, 0x20, 0x7b, 0xe4, 0xa0, 0xe1, 0xc8, 0x6f, 0xab, 0x0d, - 0xd7, 0xfe, 0x72, 0xf2, 0x41, 0x1c, 0x71, 0xa4, 0x26, 0x6c, 0xbf, 0xe1, 0x98, 0x44, 0x6e, 0x1f, - 0x95, 0x7d, 0xa6, 0xe9, 0x75, 0xd8, 0xec, 0xa0, 0x97, 0xa0, 0x30, 0x36, 0xe4, 0x1f, 0xa5, 0xac, - 0xaf, 0x04, 0x76, 0x9f, 0xa3, 0xf0, 0x7a, 0xaf, 0xe3, 0x8f, 0x18, 0x95, 0x45, 0x4d, 0xd8, 0x3e, - 0x0e, 0x03, 0x8c, 0x44, 0xfb, 0x69, 0x99, 0xa9, 0xd4, 0x33, 0xa0, 0x8d, 0x39, 0x10, 0x6d, 0x42, - 0xdd, 0x41, 0x37, 0xec, 0x1b, 0xba, 0x0c, 0x4e, 0x05, 0x35, 0x60, 0xab, 0x83, 0xc9, 0x79, 0xe0, - 0xa1, 0x51, 0x93, 0xf1, 0x52, 0x4a, 0x1a, 0x2f, 0x1e, 0x20, 0x37, 0xea, 0x7b, 0xba, 0xa4, 0x91, - 0xca, 0xf2, 0x81, 0x2e, 0xc2, 0xa8, 0x7b, 0x35, 0xa1, 0x2e, 0x03, 0x0a, 0x65, 0x2a, 0x68, 0x0b, - 0x1a, 0xcf, 0x3e, 0x0f, 0x82, 0x04, 0x79, 0x3b, 0x92, 0x30, 0xba, 0x33, 0x0f, 0x14, 0x37, 0x68, - 0x73, 0x9e, 0xa2, 0x7f, 0x28, 0x24, 0x94, 0xee, 0xcc, 0xb4, 0x75, 0x04, 0xc6, 0x0b, 0x14, 0x32, - 0xcb, 0x61, 0x2a, 0x7a, 0x71, 0x12, 0x88, 0x6c, 0x45, 0xbb, 0x8b, 0x58, 0xc7, 0x0d, 0xa7, 0x37, - 0xde, 0x71, 0xe4, 0xb7, 0xf5, 0x18, 0x6e, 0x2c, 0xc9, 0xa1, 0x80, 0x5b, 0xd0, 0x38, 0x49, 0xbb, - 0x61, 0xe0, 0xbd, 0xc2, 0x4c, 0x66, 0xda, 0x71, 0xe6, 0x01, 0xeb, 0x3d, 0xdc, 0x7c, 0x8b, 0x49, - 0x70, 0x96, 0xad, 0x4f, 0x60, 0xc0, 0xd6, 0x89, 0x9b, 0x85, 0xb1, 0xeb, 0x2b, 0x88, 0x52, 0xce, - 0xd8, 0xf4, 0x05, 0xb6, 0x47, 0xd0, 0x5a, 0x5e, 0x40, 0xe1, 0x15, 0xdd, 0x0f, 0x3e, 0x44, 0xe8, - 0x2b, 0x36, 0xa5, 0x1e, 0x7c, 0xd7, 0xa1, 0x56, 0xb8, 0xe9, 0x29, 0x5c, 0x59, 0xd8, 0x2f, 0xba, - 0x6f, 0xff, 0xbb, 0xab, 0x76, 0x75, 0x51, 0xcd, 0xdb, 0x97, 0xb8, 0x54, 0xf1, 0x77, 0x00, 0xf3, - 0x11, 0xd3, 0x5b, 0xd5, 0x43, 0x95, 0x6d, 0x34, 0xf7, 0x57, 0x9b, 0x54, 0xe2, 0x10, 0x76, 0x2b, - 0x13, 0xa1, 0x77, 0xaa, 0x47, 0xff, 0x37, 0x7a, 0xf3, 0xee, 0x5a, 0x5e, 0x55, 0x2d, 0x85, 0xe6, - 0xb2, 0x1e, 0xd3, 0x7b, 0xd5, 0x24, 0x2b, 0x86, 0x6d, 0xda, 0xeb, 0xda, 0xa7, 0x65, 0x8f, 0x9e, - 0x0c, 0x47, 0x4c, 0xbb, 0x18, 0x31, 0x6d, 0x32, 0x62, 0xe4, 0x4b, 0xce, 0xc8, 0x8f, 0x9c, 0x91, - 0x9f, 0x39, 0x23, 0xc3, 0x9c, 0x91, 0x5f, 0x39, 0x23, 0xbf, 0x73, 0xa6, 0x4d, 0x72, 0x46, 0xbe, - 0x8d, 0x99, 0x36, 0x1c, 0x33, 0xed, 0x62, 0xcc, 0xb4, 0xd3, 0x5a, 0xf1, 0xee, 0x74, 0x37, 0xe5, - 0xc3, 0xf3, 0xf0, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xba, 0xb3, 0x18, 0x70, 0x86, 0x04, 0x00, - 0x00, + // 530 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xbb, 0x8e, 0xd3, 0x40, + 0x14, 0x86, 0x3d, 0xeb, 0xec, 0xed, 0xb0, 0x05, 0x3b, 0x44, 0xc8, 0x98, 0x68, 0x14, 0x99, 0x45, + 0x8a, 0x40, 0x58, 0x02, 0x24, 0x24, 0xe8, 0xb2, 0xcb, 0x2d, 0xa2, 0x59, 0x39, 0x5c, 0xa4, 0x6d, + 0x90, 0xe3, 0x9c, 0x25, 0x16, 0x8e, 0x27, 0x78, 0xc6, 0x2b, 0xdc, 0x21, 0x5a, 0x1a, 0x1e, 0x81, + 0x92, 0x47, 0xa1, 0x4c, 0xb9, 0x25, 0x71, 0x1a, 0xca, 0x7d, 0x04, 0xe4, 0xc9, 0xe4, 0x02, 0x0e, + 0x21, 0x9d, 0xff, 0xdf, 0xff, 0x9c, 0xf3, 0x8d, 0xcf, 0x91, 0x01, 0xfc, 0x54, 0xf6, 0xdc, 0x41, + 0xc2, 0x25, 0xa7, 0x97, 0xfb, 0xbc, 0x93, 0xb9, 0xa7, 0x61, 0x84, 0x22, 0x8b, 0x03, 0xf7, 0xec, + 0xae, 0xd3, 0x00, 0x7a, 0x94, 0x60, 0x17, 0x63, 0x19, 0xfa, 0x91, 0xf0, 0xf0, 0x43, 0x8a, 0x42, + 0x52, 0x0a, 0x95, 0xe7, 0x5c, 0x48, 0x8b, 0xd4, 0x49, 0x63, 0xd7, 0x53, 0xcf, 0x4e, 0x0b, 0xae, + 0xfc, 0x91, 0x14, 0x03, 0x1e, 0x0b, 0xa4, 0x36, 0xec, 0xbc, 0x12, 0x98, 0xc4, 0x7e, 0x1f, 0x75, + 0x7c, 0xa6, 0xe9, 0x55, 0xd8, 0x6a, 0x63, 0x90, 0xa0, 0xb4, 0x36, 0xd4, 0x1b, 0xad, 0x9c, 0x2f, + 0x04, 0xf6, 0x9f, 0xa2, 0x0c, 0x7a, 0x2f, 0xf9, 0x7b, 0x8c, 0xa7, 0x4d, 0x6d, 0xd8, 0x39, 0x8a, + 0x42, 0x8c, 0x65, 0xeb, 0xf1, 0xb4, 0xd2, 0x54, 0xcf, 0x80, 0x36, 0xe6, 0x40, 0xb4, 0x0a, 0x9b, + 0x1e, 0xfa, 0x51, 0xdf, 0x32, 0x95, 0x39, 0x11, 0xd4, 0x82, 0xed, 0x36, 0x26, 0x67, 0x61, 0x80, + 0x56, 0x45, 0xf9, 0x53, 0xa9, 0x68, 0x02, 0x3e, 0x40, 0x61, 0x6d, 0xd6, 0x4d, 0x45, 0xa3, 0x94, + 0xf3, 0x99, 0x00, 0x5d, 0xa4, 0xd1, 0x17, 0xab, 0xc2, 0xa6, 0x32, 0x34, 0xcb, 0x44, 0xd0, 0x1a, + 0xec, 0x3e, 0xf9, 0x38, 0x08, 0x13, 0x14, 0xad, 0x58, 0xd1, 0x98, 0xde, 0xdc, 0x28, 0xae, 0xd0, + 0x12, 0x22, 0xc5, 0x6e, 0x53, 0x2a, 0x2a, 0xd3, 0x9b, 0xe9, 0xe2, 0x64, 0x33, 0xe6, 0x71, 0xd6, + 0xe7, 0xa9, 0xb0, 0x82, 0x3a, 0x69, 0xec, 0x78, 0x73, 0xc3, 0x39, 0x04, 0xeb, 0x19, 0x4a, 0xd5, + 0xa3, 0x99, 0xca, 0x1e, 0x4f, 0x42, 0x99, 0xad, 0x98, 0x46, 0xe1, 0xb5, 0xfd, 0x68, 0xf2, 0x41, + 0xf6, 0x3c, 0xf5, 0xec, 0x3c, 0x84, 0x6b, 0x4b, 0x6a, 0xe8, 0xeb, 0xd4, 0x60, 0xf7, 0x38, 0xed, + 0x44, 0x61, 0xf0, 0x02, 0x33, 0x55, 0x69, 0xcf, 0x9b, 0x1b, 0xce, 0x5b, 0xb8, 0xfe, 0x1a, 0x93, + 0xf0, 0x34, 0x5b, 0x9f, 0xc0, 0x82, 0xed, 0x63, 0x3f, 0x8b, 0xb8, 0xdf, 0xd5, 0x10, 0x53, 0x39, + 0x63, 0x33, 0x17, 0xd8, 0x1e, 0x40, 0x6d, 0x79, 0x03, 0x8d, 0x57, 0x0c, 0x27, 0x7c, 0x17, 0x63, + 0x57, 0xb3, 0x69, 0x75, 0xef, 0x9b, 0x09, 0x95, 0x22, 0x4d, 0x4f, 0xe0, 0xd2, 0xc2, 0xfa, 0xd1, + 0x03, 0xf7, 0xef, 0x55, 0x76, 0xcb, 0x7b, 0x6c, 0xdf, 0xfc, 0x4f, 0x4a, 0x37, 0x7f, 0x03, 0x30, + 0x5f, 0x00, 0x7a, 0xa3, 0x7c, 0xa8, 0xb4, 0xac, 0xf6, 0xc1, 0xea, 0x90, 0x2e, 0x1c, 0xc1, 0x7e, + 0x69, 0x22, 0xf4, 0x56, 0xf9, 0xe8, 0xbf, 0x46, 0x6f, 0xdf, 0x5e, 0x2b, 0xab, 0xbb, 0xa5, 0x50, + 0x5d, 0xf6, 0x8d, 0xe9, 0x9d, 0x72, 0x91, 0x15, 0xc3, 0xb6, 0xdd, 0x75, 0xe3, 0x93, 0xb6, 0x87, + 0x8f, 0x86, 0x23, 0x66, 0x9c, 0x8f, 0x98, 0x71, 0x31, 0x62, 0xe4, 0x53, 0xce, 0xc8, 0xf7, 0x9c, + 0x91, 0x1f, 0x39, 0x23, 0xc3, 0x9c, 0x91, 0x9f, 0x39, 0x23, 0xbf, 0x72, 0x66, 0x5c, 0xe4, 0x8c, + 0x7c, 0x1d, 0x33, 0x63, 0x38, 0x66, 0xc6, 0xf9, 0x98, 0x19, 0x27, 0x95, 0xe2, 0xb7, 0xd4, 0xd9, + 0x52, 0xff, 0xa5, 0xfb, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x8b, 0xda, 0x65, 0xd1, 0xa5, 0x04, + 0x00, 0x00, } func (this *CredentialsRequest) Equal(that interface{}) bool { @@ -623,6 +632,9 @@ func (this *FetchTokenResponse) Equal(that interface{}) bool { if this.IssuedAt != that1.IssuedAt { return false } + if this.Anonymous != that1.Anonymous { + return false + } return true } func (this *GetTokenAuthorityRequest) Equal(that interface{}) bool { @@ -769,11 +781,12 @@ func (this *FetchTokenResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 7) + s := make([]string, 0, 8) s = append(s, "&auth.FetchTokenResponse{") s = append(s, "Token: "+fmt.Sprintf("%#v", this.Token)+",\n") s = append(s, "ExpiresIn: "+fmt.Sprintf("%#v", this.ExpiresIn)+",\n") s = append(s, "IssuedAt: "+fmt.Sprintf("%#v", this.IssuedAt)+",\n") + s = append(s, "Anonymous: "+fmt.Sprintf("%#v", this.Anonymous)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1164,6 +1177,18 @@ func (m *FetchTokenResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Anonymous { + i-- + if m.Anonymous { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x6 + i-- + dAtA[i] = 0x98 + } if m.IssuedAt != 0 { i = encodeVarintAuth(dAtA, i, uint64(m.IssuedAt)) i-- @@ -1413,6 +1438,9 @@ func (m *FetchTokenResponse) Size() (n int) { if m.IssuedAt != 0 { n += 1 + sovAuth(uint64(m.IssuedAt)) } + if m.Anonymous { + n += 3 + } return n } @@ -1529,6 +1557,7 @@ func (this *FetchTokenResponse) String() string { `Token:` + fmt.Sprintf("%v", this.Token) + `,`, `ExpiresIn:` + fmt.Sprintf("%v", this.ExpiresIn) + `,`, `IssuedAt:` + fmt.Sprintf("%v", this.IssuedAt) + `,`, + `Anonymous:` + fmt.Sprintf("%v", this.Anonymous) + `,`, `}`, }, "") return s @@ -2089,6 +2118,26 @@ func (m *FetchTokenResponse) Unmarshal(dAtA []byte) error { break } } + case 99: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Anonymous", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAuth + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Anonymous = bool(v != 0) default: iNdEx = preIndex skippy, err := skipAuth(dAtA[iNdEx:]) diff --git a/session/auth/auth.proto b/session/auth/auth.proto index 1b4667b91..f40bbc59c 100644 --- a/session/auth/auth.proto +++ b/session/auth/auth.proto @@ -4,51 +4,49 @@ package moby.filesync.v1; option go_package = "auth"; -service Auth{ - rpc Credentials(CredentialsRequest) returns (CredentialsResponse); - rpc FetchToken(FetchTokenRequest) returns (FetchTokenResponse); - rpc GetTokenAuthority(GetTokenAuthorityRequest) returns (GetTokenAuthorityResponse); - rpc VerifyTokenAuthority(VerifyTokenAuthorityRequest) returns (VerifyTokenAuthorityResponse); +service Auth { + rpc Credentials(CredentialsRequest) returns (CredentialsResponse); + rpc FetchToken(FetchTokenRequest) returns (FetchTokenResponse); + rpc GetTokenAuthority(GetTokenAuthorityRequest) + returns (GetTokenAuthorityResponse); + rpc VerifyTokenAuthority(VerifyTokenAuthorityRequest) + returns (VerifyTokenAuthorityResponse); } -message CredentialsRequest { - string Host = 1; -} +message CredentialsRequest { string Host = 1; } message CredentialsResponse { - string Username = 1; - string Secret = 2; + string Username = 1; + string Secret = 2; } message FetchTokenRequest { - string ClientID = 1; - string Host = 2; - string Realm = 3; - string Service = 4; - repeated string Scopes = 5; + string ClientID = 1; + string Host = 2; + string Realm = 3; + string Service = 4; + repeated string Scopes = 5; } message FetchTokenResponse { - string Token = 1; - int64 ExpiresIn = 2; // seconds - int64 IssuedAt = 3; // timestamp + string Token = 1; + int64 ExpiresIn = 2; // seconds + int64 IssuedAt = 3; // timestamp + + bool Anonymous = 99; // earthly-specific } message GetTokenAuthorityRequest { - string Host = 1; - bytes Salt = 2; + string Host = 1; + bytes Salt = 2; } -message GetTokenAuthorityResponse { - bytes PublicKey = 1; -} +message GetTokenAuthorityResponse { bytes PublicKey = 1; } message VerifyTokenAuthorityRequest { - string Host = 1; - bytes Payload = 2; - bytes Salt = 3; + string Host = 1; + bytes Payload = 2; + bytes Salt = 3; } -message VerifyTokenAuthorityResponse { - bytes Signed = 1; -} +message VerifyTokenAuthorityResponse { bytes Signed = 1; } diff --git a/session/auth/authprovider/authprovider.go b/session/auth/authprovider/authprovider.go index 87618caa3..1964f2755 100644 --- a/session/auth/authprovider/authprovider.go +++ b/session/auth/authprovider/authprovider.go @@ -78,7 +78,7 @@ func (ap *authProvider) FetchToken(ctx context.Context, req *auth.FetchTokenRequ // check for statically configured bearer token if ac.RegistryToken != "" { - return toTokenResponse(ac.RegistryToken, time.Time{}, 0), nil + return toTokenResponse(ac.RegistryToken, time.Time{}, 0, false), nil } creds, err := ap.credentials(req.Host) @@ -127,19 +127,20 @@ func (ap *authProvider) FetchToken(ctx context.Context, req *auth.FetchTokenRequ if err != nil { return nil, err } - return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresIn), nil + return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresIn, false), nil } } return nil, err } - return toTokenResponse(resp.AccessToken, resp.IssuedAt, resp.ExpiresIn), nil + return toTokenResponse(resp.AccessToken, resp.IssuedAt, resp.ExpiresIn, false), nil } // do request anonymously resp, err := authutil.FetchToken(ctx, httpClient, nil, to) if err != nil { return nil, errors.Wrap(err, "failed to fetch anonymous token") } - return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresIn), nil + + return toTokenResponse(resp.Token, resp.IssuedAt, resp.ExpiresIn, true), nil } func (ap *authProvider) tlsConfig(host string) (*tls.Config, error) { @@ -276,13 +277,14 @@ func (ap *authProvider) getAuthorityKey(host string, salt []byte) (ed25519.Priva return ed25519.NewKeyFromSeed(sum[:ed25519.SeedSize]), nil } -func toTokenResponse(token string, issuedAt time.Time, expires int) *auth.FetchTokenResponse { +func toTokenResponse(token string, issuedAt time.Time, expires int, anonymous bool) *auth.FetchTokenResponse { if expires == 0 { expires = defaultExpiration } resp := &auth.FetchTokenResponse{ Token: token, ExpiresIn: int64(expires), + Anonymous: anonymous, // earthly-specific } if !issuedAt.IsZero() { resp.IssuedAt = issuedAt.Unix() From be3e419d7723b7a597f77b28d162b81fecf6c6b0 Mon Sep 17 00:00:00 2001 From: Brandon Schurman Date: Wed, 15 May 2024 15:17:35 -0400 Subject: [PATCH 48/54] feat: allow option for client to use grpc dialer Buildkit has a custom `resolveDialer` dailer, that does not support HTTP proxy. This adds the ability for a client to enable the default gRPC dialer, which allows support for HTTP. --- client/client.go | 16 +++++++++++++++- client/client_earthly.go | 11 +++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index c590ba8ab..49514fdd1 100644 --- a/client/client.go +++ b/client/client.go @@ -52,6 +52,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error grpc.WithDefaultCallOptions(grpc_retry.WithBackoff(grpc_retry.BackoffExponentialWithJitter(10*time.Millisecond, 0.1))), //earthly } needDialer := true + useDefaultDialer := false // earthly-specific var unary []grpc.UnaryClientInterceptor var stream []grpc.StreamClientInterceptor @@ -94,6 +95,11 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error headersKV = h.kv } + // earthly-specific + if _, ok := o.(*withDefaultGRPCDialer); ok { + useDefaultDialer = true + } + if opt, ok := o.(*withGRPCDialOption); ok { customDialOptions = append(customDialOptions, opt.opt) } @@ -121,7 +127,7 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error stream = append(stream, otelgrpc.StreamClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators))) } - if needDialer { + if needDialer && !useDefaultDialer { dialFn, err := resolveDialer(address) if err != nil { return nil, err @@ -164,6 +170,14 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error gopts = append(gopts, grpc.WithChainStreamInterceptor(stream...)) gopts = append(gopts, customDialOptions...) + // earthly-specific + if useDefaultDialer { + split := strings.Split(address, "://") + if len(split) > 0 { + address = split[1] + } + } + conn, err := grpc.DialContext(ctx, address, gopts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial %q . make sure buildkitd is running", address) diff --git a/client/client_earthly.go b/client/client_earthly.go index d45b8a312..4c2f788af 100644 --- a/client/client_earthly.go +++ b/client/client_earthly.go @@ -32,3 +32,14 @@ func headersStreamInterceptor(kv ...string) grpc.StreamClientInterceptor { return streamer(ctx, desc, cc, method, opts...) } } + +// WithDefaultGRPCDialer triggers the internal gRPC dialer to be used instead of the buildkit default. +// This can be important when buildkit server is behind an HTTP connect proxy, +// since the default dialer in gRPC already knows how to use those. +func WithDefaultGRPCDialer() ClientOpt { + return &withDefaultGRPCDialer{} +} + +type withDefaultGRPCDialer struct{} + +func (*withDefaultGRPCDialer) isClientOpt() {} From abb84c99a6a379d6d631b1d902df26c888772edc Mon Sep 17 00:00:00 2001 From: Mike Date: Sat, 11 May 2024 15:15:29 -0700 Subject: [PATCH 49/54] Custom inline cache implementation --- cache/remotecache/import.go | 5 +- cache/remotecache/inline.go | 253 ++++++++++++++++++++++ cache/remotecache/registry/inline.go | 58 +++++ cmd/buildkitd/main.go | 1 + control/control.go | 3 + executor/runcexecutor/monitor_stats.go | 2 +- exporter/containerimage/exptypes/types.go | 1 + exporter/containerimage/writer.go | 20 +- exporter/earthlyoutputs/export.go | 7 + solver/jobs.go | 37 ++-- solver/llbsolver/bridge.go | 120 +++++----- solver/llbsolver/inline.go | 94 ++++++++ solver/llbsolver/solver.go | 58 +++-- solver/simple.go | 244 +++++++++------------ worker/simple.go | 113 ++++++++-- 15 files changed, 768 insertions(+), 248 deletions(-) create mode 100644 cache/remotecache/inline.go create mode 100644 cache/remotecache/registry/inline.go create mode 100644 solver/llbsolver/inline.go diff --git a/cache/remotecache/import.go b/cache/remotecache/import.go index 99b9695f8..22244ea45 100644 --- a/cache/remotecache/import.go +++ b/cache/remotecache/import.go @@ -302,8 +302,9 @@ type image struct { Rootfs struct { DiffIDs []digest.Digest `json:"diff_ids"` } `json:"rootfs"` - Cache []byte `json:"moby.buildkit.cache.v0"` - History []struct { + Cache []byte `json:"moby.buildkit.cache.v0"` + EarthlyInlineCache []byte `json:"earthly.inlinecache.v0"` + History []struct { Created *time.Time `json:"created,omitempty"` CreatedBy string `json:"created_by,omitempty"` EmptyLayer bool `json:"empty_layer,omitempty"` diff --git a/cache/remotecache/inline.go b/cache/remotecache/inline.go new file mode 100644 index 000000000..fafeafa74 --- /dev/null +++ b/cache/remotecache/inline.go @@ -0,0 +1,253 @@ +package remotecache + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/labels" + v1 "github.com/moby/buildkit/cache/remotecache/v1" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/imageutil" + "github.com/moby/buildkit/util/progress" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// earthlyInlineCacheItem stores a relation between a simple solver cache key & +// a remote image descriptor. Used for inline caching. +type earthlyInlineCacheItem struct { + Key digest.Digest `json:"cacheKey"` + Descriptor digest.Digest `json:"descriptor"` +} + +// EarthlyInlineCacheRemotes produces a map of cache keys to remote sources by +// parsing inline-cache metadata from a remote image's config data. +func EarthlyInlineCacheRemotes(ctx context.Context, provider content.Provider, desc ocispecs.Descriptor, w worker.Worker) (map[digest.Digest]*solver.Remote, error) { + dt, err := readBlob(ctx, provider, desc) + if err != nil { + return nil, err + } + + manifestType, err := imageutil.DetectManifestBlobMediaType(dt) + if err != nil { + return nil, err + } + + layerDone := progress.OneOff(ctx, fmt.Sprintf("inferred cache manifest type: %s", manifestType)) + layerDone(nil) + + configDesc, err := configDescriptor(dt, manifestType) + if err != nil { + return nil, err + } + + if configDesc.Digest != "" { + return nil, errors.New("expected empty digest value") + } + + m := map[digest.Digest][]byte{} + + if err := allDistributionManifests(ctx, provider, dt, m); err != nil { + return nil, err + } + + remotes := map[digest.Digest]*solver.Remote{} + + for _, dt := range m { + var m ocispecs.Manifest + + if err := json.Unmarshal(dt, &m); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal manifest") + } + + if m.Config.Digest == "" || len(m.Layers) == 0 { + continue + } + + p, err := content.ReadBlob(ctx, provider, m.Config) + if err != nil { + return nil, errors.Wrap(err, "failed to read blob") + } + + var img image + + if err := json.Unmarshal(p, &img); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal image") + } + + if len(img.Rootfs.DiffIDs) != len(m.Layers) { + bklog.G(ctx).Warnf("invalid image with mismatching manifest and config") + continue + } + + if img.EarthlyInlineCache == nil { + continue + } + + cacheItems := []earthlyInlineCacheItem{} + if err := json.Unmarshal(img.EarthlyInlineCache, &cacheItems); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal cache items") + } + + layers, err := preprocessLayers(img, m) + if err != nil { + return nil, err + } + + found := extractRemotes(provider, cacheItems, layers) + for key, remote := range found { + remotes[key] = remote + } + } + + return remotes, nil +} + +// extractRemotes constructs a list of descriptors--which represent the layer +// chain for the given digest--for each of the items discovered in the inline +// metadata. +func extractRemotes(provider content.Provider, cacheItems []earthlyInlineCacheItem, layers []ocispecs.Descriptor) map[digest.Digest]*solver.Remote { + + remotes := map[digest.Digest]*solver.Remote{} + + for _, cacheItem := range cacheItems { + descs := []ocispecs.Descriptor{} + + found := false + for _, layer := range layers { + descs = append(descs, layer) + if layer.Digest == cacheItem.Descriptor { + found = true + break + } + } + + if found { + remote := &solver.Remote{ + Descriptors: descs, + Provider: provider, + } + + remotes[cacheItem.Key] = remote + } + } + + return remotes +} + +// preprocessLayers adds custom annotations which are used later when +// reconstructing the ref. +func preprocessLayers(img image, m ocispecs.Manifest) ([]ocispecs.Descriptor, error) { + createdDates, createdMsg, err := parseCreatedLayerInfo(img) + if err != nil { + return nil, err + } + + n := len(m.Layers) + + if len(createdDates) != n { + return nil, errors.New("unexpected creation dates length") + } + + if len(createdMsg) != n { + return nil, errors.New("unexpected creation messages length") + } + + if len(img.Rootfs.DiffIDs) != n { + return nil, errors.New("unexpected rootfs diff IDs") + } + + ret := []ocispecs.Descriptor{} + + for i, layer := range m.Layers { + if layer.Annotations == nil { + layer.Annotations = map[string]string{} + } + + if createdAt := createdDates[i]; createdAt != "" { + layer.Annotations["buildkit/createdat"] = createdAt + } + + if createdBy := createdMsg[i]; createdBy != "" { + layer.Annotations["buildkit/description"] = createdBy + } + + layer.Annotations[labels.LabelUncompressed] = img.Rootfs.DiffIDs[i].String() + + ret = append(ret, layer) + } + + return ret, nil +} + +// configDescriptor parses and returns the correct manifest for the given manifest type. +func configDescriptor(dt []byte, manifestType string) (ocispecs.Descriptor, error) { + var configDesc ocispecs.Descriptor + + switch manifestType { + case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex: + var mfst ocispecs.Index + if err := json.Unmarshal(dt, &mfst); err != nil { + return ocispecs.Descriptor{}, err + } + + for _, m := range mfst.Manifests { + if m.MediaType == v1.CacheConfigMediaTypeV0 { + configDesc = m + continue + } + } + case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest: + var mfst ocispecs.Manifest + if err := json.Unmarshal(dt, &mfst); err != nil { + return ocispecs.Descriptor{}, err + } + + if mfst.Config.MediaType == v1.CacheConfigMediaTypeV0 { + configDesc = mfst.Config + } + default: + return ocispecs.Descriptor{}, errors.Errorf("unsupported or uninferrable manifest type %s", manifestType) + } + + return configDesc, nil +} + +// allDistributionManifests pulls all manifest data & linked manifests using the provider. +func allDistributionManifests(ctx context.Context, provider content.Provider, dt []byte, m map[digest.Digest][]byte) error { + mt, err := imageutil.DetectManifestBlobMediaType(dt) + if err != nil { + return err + } + + switch mt { + case images.MediaTypeDockerSchema2Manifest, ocispecs.MediaTypeImageManifest: + m[digest.FromBytes(dt)] = dt + case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex: + var index ocispecs.Index + if err := json.Unmarshal(dt, &index); err != nil { + return errors.WithStack(err) + } + + for _, d := range index.Manifests { + if _, ok := m[d.Digest]; ok { + continue + } + p, err := content.ReadBlob(ctx, provider, d) + if err != nil { + return errors.WithStack(err) + } + if err := allDistributionManifests(ctx, provider, p, m); err != nil { + return err + } + } + } + + return nil +} diff --git a/cache/remotecache/registry/inline.go b/cache/remotecache/registry/inline.go new file mode 100644 index 000000000..267e1dca4 --- /dev/null +++ b/cache/remotecache/registry/inline.go @@ -0,0 +1,58 @@ +package registry + +import ( + "context" + "strconv" + + "github.com/containerd/containerd/remotes/docker" + "github.com/moby/buildkit/cache/remotecache" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/contentutil" + "github.com/moby/buildkit/util/resolver" + "github.com/moby/buildkit/util/resolver/limited" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// EarthlyInlineCacheRemotes fetches a group of remote sources based on values +// discovered in a remote image's inline-cache metadata field. +func EarthlyInlineCacheRemotes(ctx context.Context, sm *session.Manager, w worker.Worker, hosts docker.RegistryHosts, g session.Group, attrs map[string]string) (map[digest.Digest]*solver.Remote, error) { + ref, err := canonicalizeRef(attrs[attrRef]) + if err != nil { + return nil, err + } + + refString := ref.String() + + insecure := false + if v, ok := attrs[attrInsecure]; ok { + val, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", attrInsecure) + } + insecure = val + } + + scope, hosts := registryConfig(hosts, ref, "pull", insecure) + remote := resolver.DefaultPool.GetResolver(hosts, refString, scope, sm, g) + + xref, desc, err := remote.Resolve(ctx, refString) + if err != nil { + return nil, err + } + + fetcher, err := remote.Fetcher(ctx, xref) + if err != nil { + return nil, err + } + + src := &withDistributionSourceLabel{ + Provider: contentutil.FromFetcher(limited.Default.WrapFetcher(fetcher, refString)), + ref: refString, + source: w.ContentStore(), + } + + return remotecache.EarthlyInlineCacheRemotes(ctx, src, desc, w) +} diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 0b0d61498..c59c2cc12 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -844,6 +844,7 @@ func newController(c *cli.Context, cfg *config.Config, shutdownCh chan struct{}) ContentStore: w.ContentStore(), HistoryConfig: cfg.History, RootDir: cfg.Root, + RegistryHosts: resolverFn, }) } diff --git a/control/control.go b/control/control.go index 7b7a158e8..d3422f0aa 100644 --- a/control/control.go +++ b/control/control.go @@ -11,6 +11,7 @@ import ( contentapi "github.com/containerd/containerd/api/services/content/v1" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/services/content/contentserver" "github.com/distribution/reference" "github.com/hashicorp/go-multierror" @@ -69,6 +70,7 @@ type Opt struct { ContentStore *containerdsnapshot.Store HistoryConfig *config.HistoryConfig RootDir string + RegistryHosts docker.RegistryHosts } type Controller struct { // TODO: ControlService @@ -107,6 +109,7 @@ func NewController(opt Opt) (*Controller, error) { Entitlements: opt.Entitlements, HistoryQueue: hq, RootDir: opt.RootDir, + RegistryHosts: opt.RegistryHosts, }) if err != nil { return nil, errors.Wrap(err, "failed to create solver") diff --git a/executor/runcexecutor/monitor_stats.go b/executor/runcexecutor/monitor_stats.go index 931ae041d..e1a03e5c1 100644 --- a/executor/runcexecutor/monitor_stats.go +++ b/executor/runcexecutor/monitor_stats.go @@ -53,7 +53,7 @@ func (w *runcExecutor) monitorContainerStats(ctx context.Context, id string, sam for { select { case <-ctx.Done(): - bklog.G(ctx).Infof("stats collection context done: %v", ctx.Err()) + bklog.G(ctx).Debugf("stats collection context done: %v", ctx.Err()) return case <-timer.C: // Initial sleep will give container the chance to start. stats, err := w.runc.Stats(ctx, id) diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index c4d5721ea..83a85c379 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -11,6 +11,7 @@ const ( ExporterImageConfigDigestKey = "containerimage.config.digest" ExporterImageDescriptorKey = "containerimage.descriptor" ExporterInlineCache = "containerimage.inlinecache" + EarthlyInlineCache = "earthly.inlinecache" ExporterPlatformsKey = "refs.platforms" ) diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 0c1e91bf1..22525b579 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -138,6 +138,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session config := exptypes.ParseKey(inp.Metadata, exptypes.ExporterImageConfigKey, p) inlineCache := exptypes.ParseKey(inp.Metadata, exptypes.ExporterInlineCache, p) + earthlyInlineCache := exptypes.ParseKey(inp.Metadata, exptypes.EarthlyInlineCache, p) remote := &remotes[0] if opts.RewriteTimestamp { remote, err = ic.rewriteRemoteWithEpoch(ctx, opts, remote) @@ -145,7 +146,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return nil, err } } - mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, remote, annotations, inlineCache, opts.Epoch, session.NewGroup(sessionID)) + mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, ref, config, remote, annotations, inlineCache, earthlyInlineCache, opts.Epoch, session.NewGroup(sessionID)) if err != nil { return nil, err } @@ -203,6 +204,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } config := exptypes.ParseKey(inp.Metadata, exptypes.ExporterImageConfigKey, p) inlineCache := exptypes.ParseKey(inp.Metadata, exptypes.ExporterInlineCache, p) + earthlyInlineCache := exptypes.ParseKey(inp.Metadata, exptypes.EarthlyInlineCache, p) remote := &remotes[remotesMap[p.ID]] if remote == nil { @@ -218,7 +220,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session } } - desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCache, opts.Epoch, session.NewGroup(sessionID)) + desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, remote, opts.Annotations.Platform(&p.Platform), inlineCache, earthlyInlineCache, opts.Epoch, session.NewGroup(sessionID)) if err != nil { return nil, err } @@ -388,7 +390,7 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo }, nil } -func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *ImageCommitOpts, ref cache.ImmutableRef, config []byte, remote *solver.Remote, annotations *Annotations, inlineCache []byte, epoch *time.Time, sg session.Group) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) { +func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *ImageCommitOpts, ref cache.ImmutableRef, config []byte, remote *solver.Remote, annotations *Annotations, inlineCache, earthlyInlineCache []byte, epoch *time.Time, sg session.Group) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) { if len(config) == 0 { var err error config, err = defaultImageConfig() @@ -407,7 +409,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima return nil, nil, err } - config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, epoch) + config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, earthlyInlineCache, epoch) if err != nil { return nil, nil, err } @@ -633,7 +635,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispecs.History, error) { return config.History, nil } -func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, epoch *time.Time) ([]byte, error) { +func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache, earthlyInlineCache []byte, epoch *time.Time) ([]byte, error) { m := map[string]json.RawMessage{} if err := json.Unmarshal(dt, &m); err != nil { return nil, errors.Wrap(err, "failed to parse image config for patch") @@ -701,6 +703,14 @@ func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs m["moby.buildkit.cache.v0"] = dt } + if earthlyInlineCache != nil { + dt, err := json.Marshal(earthlyInlineCache) + if err != nil { + return nil, err + } + m["earthly.inlinecache.v0"] = dt + } + dt, err = json.Marshal(m) return dt, errors.Wrap(err, "failed to marshal config after patch") } diff --git a/exporter/earthlyoutputs/export.go b/exporter/earthlyoutputs/export.go index 06c74cb8a..c4c02df7c 100644 --- a/exporter/earthlyoutputs/export.go +++ b/exporter/earthlyoutputs/export.go @@ -268,6 +268,13 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source simpleMd[exptypes.ExporterInlineCache] = inlineCache } + // TODO: Remove the above (legacy) option. + earthlyInlineCacheK := fmt.Sprintf("%s/%s", exptypes.EarthlyInlineCache, k) + earthlyInlineCache, ok := src.Metadata[earthlyInlineCacheK] + if ok { + simpleMd[exptypes.EarthlyInlineCache] = earthlyInlineCache + } + opts := e.opts as, _, err := containerimage.ParseAnnotations(simpleMd) if err != nil { diff --git a/solver/jobs.go b/solver/jobs.go index 5116644d1..b23a853a6 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -256,37 +256,36 @@ type Job struct { } type SolverOpt struct { - ResolveOpFunc ResolveOpFunc - DefaultCache CacheManager - WorkerResultGetter workerResultGetter - CommitRefFunc CommitRefFunc - RootDir string + ResolveOpFunc ResolveOpFunc + DefaultCache CacheManager + ResultSource ResultSource + RefIDStore *RefIDStore + CommitRefFunc CommitRefFunc } func NewSolver(opts SolverOpt) *Solver { if opts.DefaultCache == nil { opts.DefaultCache = NewInMemoryCacheManager() } - jl := &Solver{ + solver := &Solver{ jobs: make(map[string]*Job), actives: make(map[digest.Digest]*state), opts: opts, index: newEdgeIndex(), } - // TODO: This should be hoisted up a few layers as not to be bound to the - // original solver. For now, we just need a convenient place to initialize - // it once. - c, err := newDiskCache(opts.WorkerResultGetter, opts.RootDir) - if err != nil { - panic(err) // TODO: Handle error appropriately once the new solver code is moved. - } - simple := newSimpleSolver(opts.ResolveOpFunc, opts.CommitRefFunc, jl, c) - jl.simple = simple - - jl.s = newScheduler(jl) - jl.updateCond = sync.NewCond(jl.mu.RLocker()) - return jl + simple := newSimpleSolver( + opts.ResolveOpFunc, + opts.CommitRefFunc, + solver, + opts.RefIDStore, + opts.ResultSource, + ) + solver.simple = simple + + solver.s = newScheduler(solver) + solver.updateCond = sync.NewCond(solver.mu.RLocker()) + return solver } func (jl *Solver) setEdge(e Edge, newEdge *edge) { diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index 8bfd96e46..e445c9539 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -8,9 +8,11 @@ import ( "time" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes/docker" "github.com/mitchellh/hashstructure/v2" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/remotecache" + "github.com/moby/buildkit/cache/remotecache/registry" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter" @@ -31,7 +33,6 @@ import ( "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" - "golang.org/x/sync/errgroup" ) type llbBridge struct { @@ -43,6 +44,10 @@ type llbBridge struct { cms map[string]solver.CacheManager cmsMu sync.Mutex sm *session.Manager + registryHosts docker.RegistryHosts + workerRemoteSource *worker.WorkerRemoteSource + importDone map[string]chan struct{} + importMu sync.Mutex } func (b *llbBridge) Warn(ctx context.Context, dgst digest.Digest, msg string, opts frontend.WarnOpts) error { @@ -78,11 +83,6 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp return nil, err } - // TODO FIXME earthly-specific wait group is required to ensure the remotecache/registry's ResolveCacheImporterFunc can run - // which requires the session to remain open in order to get dockerhub (or any other registry) credentials. - // It seems like the cleaner approach is to bake this in somewhere into the edge or Load - eg, _ := errgroup.WithContext(ctx) - srcPol, err := loadSourcePolicy(b.builder) if err != nil { return nil, err @@ -94,62 +94,13 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp } polEngine = sourcepolicy.NewEngine(pol) - if err != nil { - return nil, err - } - } - var cms []solver.CacheManager - for _, im := range cacheImports { - cmID, err := cmKey(im) - if err != nil { - return nil, err - } - b.cmsMu.Lock() - var cm solver.CacheManager - if prevCm, ok := b.cms[cmID]; !ok { - func(cmID string, im gw.CacheOptionsEntry) { - cm = newLazyCacheManager(cmID, func() (solver.CacheManager, error) { - var cmNew solver.CacheManager - if err := inBuilderContext(context.TODO(), b.builder, "importing cache manifest from "+cmID, "", func(ctx context.Context, g session.Group) error { - resolveCI, ok := b.resolveCacheImporterFuncs[im.Type] - if !ok { - return errors.Errorf("unknown cache importer: %s", im.Type) - } - ci, desc, err := resolveCI(ctx, g, im.Attrs) - if err != nil { - return errors.Wrapf(err, "failed to configure %v cache importer", im.Type) - } - cmNew, err = ci.Resolve(ctx, desc, cmID, w) - return err - }); err != nil { - bklog.G(ctx).Debugf("error while importing cache manifest from cmId=%s: %v", cmID, err) - return nil, err - } - return cmNew, nil - }) - - cmInst := cm - eg.Go(func() error { - if lcm, ok := cmInst.(*lazyCacheManager); ok { - lcm.wait() - } - return nil - }) - }(cmID, im) - b.cms[cmID] = cm - } else { - cm = prevCm - } - cms = append(cms, cm) - b.cmsMu.Unlock() - } - err = eg.Wait() - if err != nil { - return nil, err } + + b.processImports(ctx, cacheImports, w) + dpc := &detectPrunedCacheID{} - edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) + edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { return nil, errors.Wrap(err, "failed to load LLB") } @@ -173,6 +124,57 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp return res, nil } +func (b *llbBridge) processImports(ctx context.Context, cacheImports []gw.CacheOptionsEntry, w worker.Worker) { + var importRefs []string + + // Earthly custom inline cache handling. Other cache import types are ignored. + for _, cacheImport := range cacheImports { + if cacheImport.Type != "registry" { + continue + } + + importRef := cacheImport.Attrs["ref"] + importRefs = append(importRefs, importRef) + + b.importMu.Lock() + _, ok := b.importDone[importRef] + if ok { + b.importMu.Unlock() + continue + } + done := make(chan struct{}) + b.importDone[importRef] = done + b.importMu.Unlock() + + remotes := map[digest.Digest]*solver.Remote{} + name := fmt.Sprintf("importing cache manifest from %s", importRef) + + err := inBuilderContext(ctx, b.builder, name, "", func(ctx context.Context, g session.Group) error { + var err error + remotes, err = registry.EarthlyInlineCacheRemotes(ctx, b.sm, w, b.registryHosts, g, cacheImport.Attrs) + return err + }) + if err != nil { + bklog.G(ctx).Warnf("failed to import cache manifest from %s", importRef) + } + + if len(remotes) > 0 { + for cacheKey, remote := range remotes { + b.workerRemoteSource.AddResult(ctx, cacheKey, remote) + } + } + + close(done) + } + + for _, importRef := range importRefs { + b.importMu.Lock() + done := b.importDone[importRef] + b.importMu.Unlock() + <-done + } +} + // getExporter is earthly specific code which extracts the configured exporter // from the job's metadata func (b *llbBridge) getExporter(ctx context.Context) (*ExporterRequest, error) { diff --git a/solver/llbsolver/inline.go b/solver/llbsolver/inline.go new file mode 100644 index 000000000..133aa13e3 --- /dev/null +++ b/solver/llbsolver/inline.go @@ -0,0 +1,94 @@ +package llbsolver + +import ( + "context" + "encoding/json" + "fmt" + + cacheconfig "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type earthlyInlineCacheItem struct { + Key digest.Digest `json:"cacheKey"` + Descriptor digest.Digest `json:"descriptor"` +} + +// earthlyInlineCache attaches custom "inline cache" metadata which can be used +// by a new build to load image layer blobs and use them as cache results. +func earthlyInlineCache(ctx context.Context, job *solver.Job, exp exporter.ExporterInstance, cached *result.Result[solver.CachedResult]) (map[string][]byte, error) { + if cached.Ref != nil { + return nil, errors.New("unexpected ref") + } + + meta := map[string][]byte{} + + err := inBuilderContext(ctx, job, "preparing layers for inline cache", job.SessionID+"-cache-inline", func(ctx context.Context, _ session.Group) error { + for k, res := range cached.Refs { + val, err := earthlyInlineCacheDigests(ctx, job, exp, res) + if err != nil { + return err + } + meta[fmt.Sprintf("%s/%s", exptypes.EarthlyInlineCache, k)] = val + } + return nil + }) + + if err != nil { + return nil, err + } + + return meta, nil +} + +// earthlyInlineCacheDigests creates a map of computed cache keys to manifest +// layer hashes which will be used to load inline cache blobs. +func earthlyInlineCacheDigests(ctx context.Context, job *solver.Job, exp exporter.ExporterInstance, res solver.CachedResult) ([]byte, error) { + workerRef, ok := res.Sys().(*worker.WorkerRef) + if !ok { + return nil, errors.Errorf("invalid reference: %T", res.Sys()) + } + + sess := session.NewGroup(job.SessionID) + + remotes, err := workerRef.GetRemotes(ctx, true, cacheconfig.RefConfig{Compression: exp.Config().Compression()}, false, sess) + if err != nil || len(remotes) == 0 { + return nil, nil + } + + var ( + remote = remotes[0] + cacheItems = []earthlyInlineCacheItem{} + cacheKeys = res.CacheKeys() + ) + + for i := 0; i < len(cacheKeys) && i < len(remote.Descriptors); i++ { + cacheItems = append(cacheItems, earthlyInlineCacheItem{ + Key: cacheKeys[i].Digest(), + Descriptor: remote.Descriptors[i].Digest, + }) + } + + val, err := json.Marshal(cacheItems) + if err != nil { + return nil, err + } + + return val, nil +} + +func hasInlineCacheExporter(exporters []RemoteCacheExporter) bool { + for _, exp := range exporters { + if _, ok := asInlineCache(exp.Exporter); ok { + return true + } + } + return false +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 5cc90aa5f..12dae8052 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/containerd/containerd/remotes/docker" intoto "github.com/in-toto/in-toto-golang/in_toto" slsa02 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" controlapi "github.com/moby/buildkit/api/services/control" @@ -80,6 +81,7 @@ type Opt struct { HistoryQueue *HistoryQueue ResourceMonitor *resources.Monitor RootDir string + RegistryHosts docker.RegistryHosts } type Solver struct { @@ -94,6 +96,8 @@ type Solver struct { entitlements []string history *HistoryQueue sysSampler *resources.Sampler[*resourcetypes.SysSample] + registryHosts docker.RegistryHosts + workerRemoteSource *worker.WorkerRemoteSource } // Processor defines a processing function to be applied after solving, but @@ -101,6 +105,13 @@ type Solver struct { type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job, usage *resources.SysSampler) (*Result, error) func New(opt Opt) (*Solver, error) { + defaultWorker, err := opt.WorkerController.GetDefault() + if err != nil { + return nil, err + } + + remoteSource := worker.NewWorkerRemoteSource(defaultWorker) + s := &Solver{ workerController: opt.WorkerController, resolveWorker: defaultResolver(opt.WorkerController), @@ -111,6 +122,8 @@ func New(opt Opt) (*Solver, error) { sm: opt.SessionManager, entitlements: opt.Entitlements, history: opt.HistoryQueue, + registryHosts: opt.RegistryHosts, + workerRemoteSource: remoteSource, } sampler, err := resources.NewSysSampler() @@ -119,12 +132,22 @@ func New(opt Opt) (*Solver, error) { } s.sysSampler = sampler + refIDStore, err := solver.NewRefIDStore(opt.RootDir) + if err != nil { + return nil, err + } + + sources := worker.NewCombinedResultSource( + worker.NewWorkerResultSource(opt.WorkerController, refIDStore), + remoteSource, + ) + s.solver = solver.NewSolver(solver.SolverOpt{ - ResolveOpFunc: s.resolver(), - DefaultCache: opt.CacheManager, - WorkerResultGetter: worker.NewWorkerResultGetter(opt.WorkerController), - CommitRefFunc: worker.FinalizeRef, - RootDir: opt.RootDir, + ResolveOpFunc: s.resolver(), + DefaultCache: opt.CacheManager, + ResultSource: sources, + CommitRefFunc: worker.FinalizeRef, + RefIDStore: refIDStore, }) return s, nil } @@ -148,6 +171,10 @@ func (s *Solver) bridge(b solver.Builder) *provenanceBridge { resolveCacheImporterFuncs: s.resolveCacheImporterFuncs, cms: map[string]solver.CacheManager{}, sm: s.sm, + registryHosts: s.registryHosts, + workerRemoteSource: s.workerRemoteSource, + importDone: map[string]chan struct{}{}, + importMu: sync.Mutex{}, }} } @@ -557,16 +584,17 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - cacheExporters, inlineCacheExporter := splitCacheExporters(exp.CacheExporters) - + cacheExporters, _ := splitCacheExporters(exp.CacheExporters) var exporterResponse map[string]string if e := exp.Exporter; e != nil { - meta, err := runInlineCacheExporter(ctx, e, inlineCacheExporter, j, cached) - if err != nil { - return nil, err - } - for k, v := range meta { - inp.AddMeta(k, v) + if hasInlineCacheExporter(exp.CacheExporters) { + meta, err := earthlyInlineCache(ctx, j, e, cached) + if err != nil { + return nil, errors.Wrap(err, "failed prepare inline cache") + } + for k, v := range meta { + inp.AddMeta(k, v) + } } if err := inBuilderContext(ctx, j, e.Name(), j.SessionID+"-export", func(ctx context.Context, _ session.Group) error { @@ -577,6 +605,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } } + // Deprecated. Can be removed later. cacheExporterResponse, err := runCacheExporters(ctx, cacheExporters, j, cached, inp) if err != nil { return nil, err @@ -602,6 +631,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro }, nil } +// Deprecated. Can be removed later. func runCacheExporters(ctx context.Context, exporters []RemoteCacheExporter, j *solver.Job, cached *result.Result[solver.CachedResult], inp *result.Result[cache.ImmutableRef]) (map[string]string, error) { eg, ctx := errgroup.WithContext(ctx) g := session.NewGroup(j.SessionID) @@ -654,6 +684,7 @@ func runCacheExporters(ctx context.Context, exporters []RemoteCacheExporter, j * return cacheExporterResponse, nil } +// Deprecated. Can be removed later. func runInlineCacheExporter(ctx context.Context, e exporter.ExporterInstance, inlineExporter *RemoteCacheExporter, j *solver.Job, cached *result.Result[solver.CachedResult]) (map[string][]byte, error) { meta := map[string][]byte{} if inlineExporter == nil { @@ -835,6 +866,7 @@ func asInlineCache(e remotecache.Exporter) (inlineCacheExporter, bool) { return ie, ok } +// Deprecated. Can be removed later. func inlineCache(ctx context.Context, e remotecache.Exporter, res solver.CachedResult, compressionopt compression.Config, g session.Group) ([]byte, error) { ie, ok := asInlineCache(e) if !ok { diff --git a/solver/simple.go b/solver/simple.go index 4ecb3e369..274545d38 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -18,29 +18,41 @@ import ( bolt "go.etcd.io/bbolt" ) -var ErrRefNotFound = errors.New("ref not found") - // CommitRefFunc can be used to finalize a Result's ImmutableRef. type CommitRefFunc func(ctx context.Context, result Result) error +// ResultSource can be any source (local or remote) that allows one to load a +// Result using a cache key digest. +type ResultSource interface { + Load(ctx context.Context, cacheKey digest.Digest) (Result, bool, error) +} + type simpleSolver struct { resolveOpFunc ResolveOpFunc commitRefFunc CommitRefFunc solver *Solver parallelGuard *parallelGuard - resultCache resultCache + refIDStore *RefIDStore + resultSource ResultSource cacheKeyManager *cacheKeyManager mu sync.Mutex } -func newSimpleSolver(resolveOpFunc ResolveOpFunc, commitRefFunc CommitRefFunc, solver *Solver, cache resultCache) *simpleSolver { +func newSimpleSolver( + resolveOpFunc ResolveOpFunc, + commitRefFunc CommitRefFunc, + solver *Solver, + refIDStore *RefIDStore, + resultSource ResultSource, +) *simpleSolver { return &simpleSolver{ cacheKeyManager: newCacheKeyManager(), - resultCache: cache, parallelGuard: newParallelGuard(time.Millisecond * 100), resolveOpFunc: resolveOpFunc, commitRefFunc: commitRefFunc, solver: solver, + refIDStore: refIDStore, + resultSource: resultSource, } } @@ -49,7 +61,8 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul // Ordered list of vertices to build. digests, vertices := s.exploreVertices(e) - var ret CachedResult + var ret Result + var expKeys []ExportableCacheKey for _, d := range digests { vertex, ok := vertices[d] @@ -57,20 +70,29 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return nil, errors.Errorf("digest %s not found", d) } - res, expCacheKeys, err := s.buildOne(ctx, d, vertex, job, e) + res, cacheKey, err := s.buildOne(ctx, d, vertex, job, e) if err != nil { return nil, err } - ret = NewCachedResult(res, expCacheKeys) + ret = res + + // Hijack the CacheKey type in order to export a reference from the new cache key to the ref ID. + expKeys = append(expKeys, ExportableCacheKey{ + CacheKey: &CacheKey{ + ID: res.ID(), + digest: cacheKey, + }, + Exporter: nil, // We're not using an exporter here. + }) } - return ret, nil + return NewCachedResult(ret, expKeys), nil } -func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, []ExportableCacheKey, error) { +func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, digest.Digest, error) { // Ensure we don't have multiple threads working on the same digest. - wait, done := s.parallelGuard.acquire(ctx, d.String()) + wait, done := s.parallelGuard.acquire(ctx, d) defer done() <-wait @@ -82,37 +104,33 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver // CacheMap populates required fields in SourceOp. cm, err := st.op.CacheMap(ctx, int(e.Index)) if err != nil { - return nil, nil, err + return nil, "", err } inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, job) if err != nil { notifyError(ctx, st, false, err) - return nil, nil, err + return nil, "", err } - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d.String()) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d) if err != nil { - return nil, nil, err + return nil, "", err } - v, ok, err := s.resultCache.get(ctx, cacheKey) + v, ok, err := s.resultSource.Load(ctx, cacheKey) if err != nil { - return nil, nil, err - } - - expCacheKeys := []ExportableCacheKey{ - {Exporter: &simpleExporter{cacheKey: cacheKey}}, + return nil, "", err } if ok && v != nil { notifyError(ctx, st, true, nil) - return v, expCacheKeys, nil + return v, cacheKey, nil } results, _, err := st.op.Exec(ctx, inputs) if err != nil { - return nil, nil, err + return nil, "", err } // Ensure all results are finalized (committed to cache). It may be better @@ -120,18 +138,18 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver for _, res := range results { err = s.commitRefFunc(ctx, res) if err != nil { - return nil, nil, err + return nil, "", err } } res := results[int(e.Index)] - err = s.resultCache.set(ctx, cacheKey, res) + err = s.refIDStore.Set(ctx, cacheKey, res.ID()) if err != nil { - return nil, nil, err + return nil, "", err } - return res, expCacheKeys, nil + return res, cacheKey, nil } func notifyError(ctx context.Context, st *state, cached bool, err error) { @@ -222,9 +240,9 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. scm := simpleCacheMap{ - digest: cm.Digest.String(), + digest: cm.Digest, deps: make([]cacheMapDep, len(cm.Deps)), - inputs: make([]string, len(cm.Deps)), + inputs: make([]digest.Digest, len(cm.Deps)), } // By default we generate a cache key that's not salted as the keys need to @@ -239,22 +257,22 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V for i, in := range vertex.Inputs() { - digest := in.Vertex.Digest().String() + d := in.Vertex.Digest() // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, digest) + cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d) if err != nil { return nil, err } // Lookup the result for that cache key. - res, ok, err := s.resultCache.get(ctx, cacheKey) + res, ok, err := s.resultSource.Load(ctx, cacheKey) if err != nil { return nil, err } if !ok { - return nil, errors.Errorf("result not found for digest: %s", digest) + return nil, errors.Errorf("result not found for digest: %s", d) } dep := cm.Deps[i] @@ -269,7 +287,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V // Add selectors (usually file references) to the struct. scm.deps[i] = cacheMapDep{ - selector: dep.Selector.String(), + selector: dep.Selector, } // ComputeDigestFunc will usually checksum files. This is then used as @@ -281,66 +299,66 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V bklog.G(ctx).Warnf("failed to compute digest: %v", err) return nil, err } else { - scm.deps[i].computed = compDigest.String() + scm.deps[i].computed = compDigest } } // Add input references to the struct as to link dependencies. - scm.inputs[i] = in.Vertex.Digest().String() + scm.inputs[i] = in.Vertex.Digest() // Add the cached result to the input set. These inputs are used to // reconstruct dependencies (mounts, etc.) for a new container run. inputs = append(inputs, res) } - s.cacheKeyManager.add(vertex.Digest().String(), &scm) + s.cacheKeyManager.add(vertex.Digest(), &scm) return inputs, nil } type cacheKeyManager struct { - cacheMaps map[string]*simpleCacheMap + cacheMaps map[digest.Digest]*simpleCacheMap mu sync.Mutex } type cacheMapDep struct { - selector string - computed string + selector digest.Digest + computed digest.Digest } type simpleCacheMap struct { - digest string - inputs []string + digest digest.Digest + inputs []digest.Digest deps []cacheMapDep salt string } func newCacheKeyManager() *cacheKeyManager { return &cacheKeyManager{ - cacheMaps: map[string]*simpleCacheMap{}, + cacheMaps: map[digest.Digest]*simpleCacheMap{}, } } -func (m *cacheKeyManager) add(key string, s *simpleCacheMap) { +func (m *cacheKeyManager) add(d digest.Digest, s *simpleCacheMap) { m.mu.Lock() - m.cacheMaps[key] = s + m.cacheMaps[d] = s m.mu.Unlock() } // cacheKey recursively generates a cache key based on a sequence of ancestor // operations & their cacheable values. -func (m *cacheKeyManager) cacheKey(ctx context.Context, digest string) (string, error) { +func (m *cacheKeyManager) cacheKey(ctx context.Context, d digest.Digest) (digest.Digest, error) { h := sha256.New() - err := m.cacheKeyRecurse(ctx, digest, h) + err := m.cacheKeyRecurse(ctx, d, h) if err != nil { return "", err } - return fmt.Sprintf("%x", h.Sum(nil)), nil + return newDigest(fmt.Sprintf("%x", h.Sum(nil))), nil } -func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash.Hash) error { +func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d digest.Digest, h hash.Hash) error { m.mu.Lock() c, ok := m.cacheMaps[d] m.mu.Unlock() @@ -359,13 +377,13 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash. } } - io.WriteString(h, c.digest) + io.WriteString(h, c.digest.String()) for _, dep := range c.deps { if dep.selector != "" { - io.WriteString(h, dep.selector) + io.WriteString(h, dep.selector.String()) } if dep.computed != "" { - io.WriteString(h, dep.computed) + io.WriteString(h, dep.computed.String()) } } @@ -374,15 +392,15 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d string, h hash. type parallelGuard struct { wait time.Duration - active map[string]struct{} + active map[digest.Digest]struct{} mu sync.Mutex } func newParallelGuard(wait time.Duration) *parallelGuard { - return ¶llelGuard{wait: wait, active: map[string]struct{}{}} + return ¶llelGuard{wait: wait, active: map[digest.Digest]struct{}{}} } -func (f *parallelGuard) acquire(ctx context.Context, d string) (<-chan struct{}, func()) { +func (f *parallelGuard) acquire(ctx context.Context, d digest.Digest) (<-chan struct{}, func()) { ch := make(chan struct{}) @@ -426,62 +444,30 @@ func (f *parallelGuard) acquire(ctx context.Context, d string) (<-chan struct{}, return ch, closer } -type resultCache interface { - set(ctx context.Context, key string, r Result) error - get(ctx context.Context, key string) (Result, bool, error) -} - -type inMemCache struct { - cache map[string]Result - mu sync.Mutex -} - -func newInMemCache() *inMemCache { - return &inMemCache{cache: map[string]Result{}} +// RefIDStore uses a BoltDB database to store links from computed cache keys to +// worker ref IDs. +type RefIDStore struct { + db *bolt.DB + bucketName string + rootDir string } -func (c *inMemCache) set(ctx context.Context, key string, r Result) error { - c.mu.Lock() - c.cache[key] = r - c.mu.Unlock() - return nil -} - -func (c *inMemCache) get(ctx context.Context, key string) (Result, bool, error) { - c.mu.Lock() - r, ok := c.cache[key] - c.mu.Unlock() - return r, ok, nil -} - -var _ resultCache = &inMemCache{} - -type diskCache struct { - resultGetter workerResultGetter - db *bolt.DB - bucketName string - rootDir string -} - -type workerResultGetter interface { - Get(ctx context.Context, id string) (Result, error) -} - -func newDiskCache(resultGetter workerResultGetter, rootDir string) (*diskCache, error) { - c := &diskCache{ - bucketName: "ids", - resultGetter: resultGetter, - rootDir: rootDir, +// NewRefIDStore creates and returns a new store and initializes a BoltDB +// instance in the specified root directory. +func NewRefIDStore(rootDir string) (*RefIDStore, error) { + r := &RefIDStore{ + bucketName: "ids", + rootDir: rootDir, } - err := c.init() + err := r.init() if err != nil { return nil, err } - return c, nil + return r, nil } -func (c *diskCache) init() error { - db, err := bolt.Open(filepath.Join(c.rootDir, "ids.db"), 0755, nil) +func (r *RefIDStore) init() error { + db, err := bolt.Open(filepath.Join(r.rootDir, "ids.db"), 0755, nil) if err != nil { return err } @@ -492,56 +478,42 @@ func (c *diskCache) init() error { if err != nil { return err } - c.db = db + r.db = db return nil } -func (c *diskCache) set(ctx context.Context, key string, r Result) error { - return c.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(c.bucketName)) - return b.Put([]byte(key), []byte(r.ID())) +// Set a cache key digest to the value of the worker ref ID. +func (r *RefIDStore) Set(ctx context.Context, key digest.Digest, id string) error { + return r.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucketName)) + return b.Put([]byte(key), []byte(id)) }) } -func (c *diskCache) get(ctx context.Context, key string) (Result, bool, error) { +// Get a worker ref ID given a cache key digest. +func (r *RefIDStore) Get(ctx context.Context, cacheKey digest.Digest) (string, bool, error) { var id string - err := c.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(c.bucketName)) - id = string(b.Get([]byte(key))) + err := r.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucketName)) + id = string(b.Get([]byte(cacheKey))) return nil }) if err != nil { - return nil, false, err + return "", false, err } if id == "" { - return nil, false, nil + return "", false, nil } - res, err := c.resultGetter.Get(ctx, id) - if err != nil { - if errors.Is(err, ErrRefNotFound) { - if err := c.delete(ctx, key); err != nil { - bklog.G(ctx).Warnf("failed to delete cache key: %v", err) - } - return nil, false, nil - } - return nil, false, err - } - return res, true, nil + return id, true, nil } -func (c *diskCache) delete(_ context.Context, key string) error { - return c.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(c.bucketName)) +func (r *RefIDStore) delete(_ context.Context, key string) error { + return r.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucketName)) return b.Delete([]byte(key)) }) } -var _ resultCache = &diskCache{} - -type simpleExporter struct { - cacheKey string -} - -func (s *simpleExporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt CacheExportOpt) ([]CacheExporterRecord, error) { - return nil, nil +func newDigest(s string) digest.Digest { + return digest.NewDigestFromEncoded(digest.SHA256, s) } diff --git a/worker/simple.go b/worker/simple.go index dec48e1d3..a5a4a4344 100644 --- a/worker/simple.go +++ b/worker/simple.go @@ -2,47 +2,66 @@ package worker import ( "context" + "sync" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" + digest "github.com/opencontainers/go-digest" ) -// WorkerResultGetter abstracts the work involved in loading a Result from a +// RefIDSource allows the caller to translate between a cache key and a worker ref ID. +type RefIDSource interface { + Get(ctx context.Context, cacheKey digest.Digest) (string, bool, error) +} + +// WorkerResultSource abstracts the work involved in loading a Result from a // worker using a ref ID. -type WorkerResultGetter struct { - wc *Controller +type WorkerResultSource struct { + wc *Controller + ids RefIDSource } -// NewWorkerResultGetter creates and returns a new *WorkerResultGetter. -func NewWorkerResultGetter(wc *Controller) *WorkerResultGetter { - return &WorkerResultGetter{wc: wc} +// NewWorkerResultSource creates and returns a new *WorkerResultSource. +func NewWorkerResultSource(wc *Controller, ids RefIDSource) *WorkerResultSource { + return &WorkerResultSource{wc: wc, ids: ids} } -// Get a cached results from a worker. -func (w *WorkerResultGetter) Get(ctx context.Context, id string) (solver.Result, error) { +// Load a cached result from a worker. +func (w *WorkerResultSource) Load(ctx context.Context, cacheKey digest.Digest) (solver.Result, bool, error) { + id, ok, err := w.ids.Get(ctx, cacheKey) + if err != nil { + return nil, false, err + } + + if !ok { + return nil, false, nil + } + workerID, refID, err := parseWorkerRef(id) if err != nil { - return nil, err + return nil, false, err } worker, err := w.wc.Get(workerID) if err != nil { - return nil, err + return nil, false, err } ref, err := worker.LoadRef(ctx, refID, false) if err != nil { if cache.IsNotFound(err) { bklog.G(ctx).Warnf("could not load ref from worker: %v", err) - return nil, solver.ErrRefNotFound + return nil, false, nil } - return nil, err + return nil, false, err } - return NewWorkerRefResult(ref, worker), nil + return NewWorkerRefResult(ref, worker), true, nil } +var _ solver.ResultSource = &WorkerResultSource{} + // FinalizeRef is a convenience function that calls Finalize on a Result's // ImmutableRef. The 'worker' package cannot be imported by 'solver' due to an // import cycle, so this function is passed in with solver.SolverOpt. @@ -56,3 +75,71 @@ func FinalizeRef(ctx context.Context, res solver.Result) error { } return nil } + +// WorkerRemoteSource can be used to fetch a remote worker source. +type WorkerRemoteSource struct { + worker Worker + remotes map[digest.Digest]*solver.Remote + mu sync.Mutex +} + +// NewWorkerRemoteSource creates and returns a remote result source. +func NewWorkerRemoteSource(worker Worker) *WorkerRemoteSource { + return &WorkerRemoteSource{ + worker: worker, + remotes: map[digest.Digest]*solver.Remote{}, + } +} + +// Load a Result from the worker. +func (w *WorkerRemoteSource) Load(ctx context.Context, cacheKey digest.Digest) (solver.Result, bool, error) { + w.mu.Lock() + remote, ok := w.remotes[cacheKey] + w.mu.Unlock() + + if !ok { + return nil, false, nil + } + + ref, err := w.worker.FromRemote(ctx, remote) + if err != nil { + return nil, false, err + } + + return NewWorkerRefResult(ref, w.worker), true, nil +} + +// AddResult adds a solver.Remote source for the given cache key. +func (w *WorkerRemoteSource) AddResult(ctx context.Context, cacheKey digest.Digest, remote *solver.Remote) { + w.mu.Lock() + defer w.mu.Unlock() + w.remotes[cacheKey] = remote +} + +var _ solver.ResultSource = &WorkerRemoteSource{} + +// CombinedResultSource implements solver.ResultSource over a list of sources. +type CombinedResultSource struct { + sources []solver.ResultSource +} + +// NewCombinedResultSource creates and returns a new source from a list of sources. +func NewCombinedResultSource(sources ...solver.ResultSource) *CombinedResultSource { + return &CombinedResultSource{sources: sources} +} + +// Load attempts to load a Result from all underlying sources. +func (c *CombinedResultSource) Load(ctx context.Context, cacheKey digest.Digest) (solver.Result, bool, error) { + for _, source := range c.sources { + res, ok, err := source.Load(ctx, cacheKey) + if err != nil { + return nil, false, err + } + if ok { + return res, true, nil + } + } + return nil, false, nil +} + +var _ solver.ResultSource = &CombinedResultSource{} From d4e630c482bde6e4f6da3d2257ee6e8d9955f180 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 17 May 2024 15:00:11 -0700 Subject: [PATCH 50/54] Lock ops based on computed key rather than LLB digest --- solver/simple.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 274545d38..5fa347c5b 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -91,11 +91,6 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul } func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, digest.Digest, error) { - // Ensure we don't have multiple threads working on the same digest. - wait, done := s.parallelGuard.acquire(ctx, d) - defer done() - <-wait - st := s.state(vertex, job) // Add cache opts to context as they will be accessed by cache retrieval. @@ -118,6 +113,14 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, "", err } + // Ensure we don't have multiple threads working on the same operation. The + // computed cache key needs to be used here instead of the vertex + // digest. This is because the vertex can sometimes differ for the same + // operation depending on its ancestors. + wait, done := s.parallelGuard.acquire(ctx, cacheKey) + defer done() + <-wait + v, ok, err := s.resultSource.Load(ctx, cacheKey) if err != nil { return nil, "", err From d144e62af27554e266fd0c4f824f7323f90c2ff6 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 17 May 2024 15:00:11 -0700 Subject: [PATCH 51/54] Lock ops based on computed key rather than LLB digest --- solver/jobs.go | 4 +- solver/llbsolver/simple.go | 29 +++++++++++ solver/llbsolver/solver.go | 1 + solver/simple.go | 104 ++++++++++++++++++++++++++++++++----- 4 files changed, 123 insertions(+), 15 deletions(-) create mode 100644 solver/llbsolver/simple.go diff --git a/solver/jobs.go b/solver/jobs.go index b23a853a6..7fa06be10 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -20,7 +20,7 @@ import ( "go.opentelemetry.io/otel/trace" ) -// ResolveOpFunc finds an Op implementation for a Vertex +// ResolveOpFunc finds an Op implementation for a Vertex. type ResolveOpFunc func(Vertex, Builder) (Op, error) type Builder interface { @@ -261,6 +261,7 @@ type SolverOpt struct { ResultSource ResultSource RefIDStore *RefIDStore CommitRefFunc CommitRefFunc + IsRunOnceFunc IsRunOnceFunc } func NewSolver(opts SolverOpt) *Solver { @@ -280,6 +281,7 @@ func NewSolver(opts SolverOpt) *Solver { solver, opts.RefIDStore, opts.ResultSource, + opts.IsRunOnceFunc, ) solver.simple = simple diff --git a/solver/llbsolver/simple.go b/solver/llbsolver/simple.go new file mode 100644 index 000000000..3646c5677 --- /dev/null +++ b/solver/llbsolver/simple.go @@ -0,0 +1,29 @@ +package llbsolver + +import ( + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/ops" +) + +// isRunOnce returns a function that can be called to determine if a Vertex +// contains an operation that must be run at least once per build. +func (s *Solver) isRunOnceOp() solver.IsRunOnceFunc { + return func(v solver.Vertex, b solver.Builder) (bool, error) { + w, err := s.resolveWorker() + if err != nil { + return false, err + } + + op, err := w.ResolveOp(v, s.Bridge(b), s.sm) + if err != nil { + return false, err + } + + switch op.(type) { + case *ops.SourceOp: + return true, nil + default: + return false, nil + } + } +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 12dae8052..4c228e354 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -144,6 +144,7 @@ func New(opt Opt) (*Solver, error) { s.solver = solver.NewSolver(solver.SolverOpt{ ResolveOpFunc: s.resolver(), + IsRunOnceFunc: s.isRunOnceOp(), DefaultCache: opt.CacheManager, ResultSource: sources, CommitRefFunc: worker.FinalizeRef, diff --git a/solver/simple.go b/solver/simple.go index 5fa347c5b..813ee144d 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/hashicorp/golang-lru/simplelru" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/tracing" @@ -18,24 +19,63 @@ import ( bolt "go.etcd.io/bbolt" ) +const ( + runOnceLRUSize = 2000 + parallelGuardWait = time.Millisecond * 100 +) + // CommitRefFunc can be used to finalize a Result's ImmutableRef. type CommitRefFunc func(ctx context.Context, result Result) error +// IsRunOnceFunc determines if the vertex represents an operation that needs to +// be run at least once. +type IsRunOnceFunc func(Vertex, Builder) (bool, error) + // ResultSource can be any source (local or remote) that allows one to load a // Result using a cache key digest. type ResultSource interface { Load(ctx context.Context, cacheKey digest.Digest) (Result, bool, error) } +// runOnceCtrl is a simple wrapper around an LRU cache. It's used to ensure that +// an operation is only run once per job. However, this is not guaranteed, as +// the struct uses a reasonable small LRU size to preview excessive memory use. +type runOnceCtrl struct { + lru *simplelru.LRU + mu sync.Mutex +} + +func newRunOnceCtrl() *runOnceCtrl { + lru, _ := simplelru.NewLRU(runOnceLRUSize, nil) // Error impossible on positive first argument. + return &runOnceCtrl{lru: lru} +} + +// hasRun: Here, we use an LRU cache to whether we need to execute the source +// operation for this job. The jobs may be re-run if the LRU size is exceeded, +// but this shouldn't have a big impact on the build. The trade-off is +// worthwhile given the memory-friendliness of LRUs. +func (s *runOnceCtrl) hasRun(d digest.Digest, sessionID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + key := fmt.Sprintf("%s:%s", sessionID, d) + ret := s.lru.Contains(key) + + s.lru.Add(key, struct{}{}) + + return ret +} + type simpleSolver struct { resolveOpFunc ResolveOpFunc + isRunOnceFunc IsRunOnceFunc commitRefFunc CommitRefFunc solver *Solver parallelGuard *parallelGuard refIDStore *RefIDStore resultSource ResultSource cacheKeyManager *cacheKeyManager - mu sync.Mutex + runOnceCtrl *runOnceCtrl } func newSimpleSolver( @@ -44,15 +84,18 @@ func newSimpleSolver( solver *Solver, refIDStore *RefIDStore, resultSource ResultSource, + isRunOnceFunc IsRunOnceFunc, ) *simpleSolver { return &simpleSolver{ cacheKeyManager: newCacheKeyManager(), - parallelGuard: newParallelGuard(time.Millisecond * 100), + parallelGuard: newParallelGuard(parallelGuardWait), resolveOpFunc: resolveOpFunc, commitRefFunc: commitRefFunc, solver: solver, refIDStore: refIDStore, resultSource: resultSource, + isRunOnceFunc: isRunOnceFunc, + runOnceCtrl: newRunOnceCtrl(), } } @@ -75,6 +118,14 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return nil, err } + // Release previous result as this is not the final return value. + if ret != nil { + err := ret.Release(ctx) + if err != nil { + return nil, err + } + } + ret = res // Hijack the CacheKey type in order to export a reference from the new cache key to the ref ID. @@ -87,6 +138,11 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul }) } + err := s.commitRefFunc(ctx, ret) + if err != nil { + return nil, err + } + return NewCachedResult(ret, expKeys), nil } @@ -121,14 +177,25 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver defer done() <-wait - v, ok, err := s.resultSource.Load(ctx, cacheKey) + isRunOnce, err := s.isRunOnceFunc(vertex, job) if err != nil { return nil, "", err } - if ok && v != nil { - notifyError(ctx, st, true, nil) - return v, cacheKey, nil + // Special case for source operations. They need to be run once per build or + // content changes will not be reliably detected. + mayLoadCache := !isRunOnce || isRunOnce && s.runOnceCtrl.hasRun(cacheKey, job.SessionID) + + if mayLoadCache { + v, ok, err := s.resultSource.Load(ctx, cacheKey) + if err != nil { + return nil, "", err + } + + if ok && v != nil { + notifyError(ctx, st, true, nil) + return v, cacheKey, nil + } } results, _, err := st.op.Exec(ctx, inputs) @@ -136,17 +203,17 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, "", err } - // Ensure all results are finalized (committed to cache). It may be better - // to background these calls at some point. - for _, res := range results { - err = s.commitRefFunc(ctx, res) - if err != nil { - return nil, "", err + res := results[int(e.Index)] + + for i := range results { + if i != int(e.Index) { + err = results[i].Release(ctx) + if err != nil { + return nil, "", err + } } } - res := results[int(e.Index)] - err = s.refIDStore.Set(ctx, cacheKey, res.ID()) if err != nil { return nil, "", err @@ -306,6 +373,15 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V } } + // The result can be released now that the preprocess & slow cache + // digest functions have been run. This is crucial as failing to do so + // will lead to full file copying from previously executed source + // operations. + err = res.Release(ctx) + if err != nil { + return nil, err + } + // Add input references to the struct as to link dependencies. scm.inputs[i] = in.Vertex.Digest() From 7d8c91775530a97f9e41c5b39f32a7d974caac7d Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 23 May 2024 10:15:46 -0700 Subject: [PATCH 52/54] Prune ref ID database & initialize cache key manager per build --- solver/jobs.go | 2 - solver/llbsolver/solver.go | 5 +- solver/simple.go | 124 ++++--------------- worker/simple.go | 241 +++++++++++++++++++++++++++++++++++-- 4 files changed, 257 insertions(+), 115 deletions(-) diff --git a/solver/jobs.go b/solver/jobs.go index 7fa06be10..f6bf52c70 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -259,7 +259,6 @@ type SolverOpt struct { ResolveOpFunc ResolveOpFunc DefaultCache CacheManager ResultSource ResultSource - RefIDStore *RefIDStore CommitRefFunc CommitRefFunc IsRunOnceFunc IsRunOnceFunc } @@ -279,7 +278,6 @@ func NewSolver(opts SolverOpt) *Solver { opts.ResolveOpFunc, opts.CommitRefFunc, solver, - opts.RefIDStore, opts.ResultSource, opts.IsRunOnceFunc, ) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 4c228e354..285f69c26 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -132,13 +132,13 @@ func New(opt Opt) (*Solver, error) { } s.sysSampler = sampler - refIDStore, err := solver.NewRefIDStore(opt.RootDir) + workerSource, err := worker.NewWorkerResultSource(opt.WorkerController, opt.RootDir) if err != nil { return nil, err } sources := worker.NewCombinedResultSource( - worker.NewWorkerResultSource(opt.WorkerController, refIDStore), + workerSource, remoteSource, ) @@ -148,7 +148,6 @@ func New(opt Opt) (*Solver, error) { DefaultCache: opt.CacheManager, ResultSource: sources, CommitRefFunc: worker.FinalizeRef, - RefIDStore: refIDStore, }) return s, nil } diff --git a/solver/simple.go b/solver/simple.go index 813ee144d..752ba3d98 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -6,7 +6,6 @@ import ( "fmt" "hash" "io" - "path/filepath" "sync" "time" @@ -16,7 +15,6 @@ import ( "github.com/moby/buildkit/util/tracing" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" ) const ( @@ -35,6 +33,7 @@ type IsRunOnceFunc func(Vertex, Builder) (bool, error) // Result using a cache key digest. type ResultSource interface { Load(ctx context.Context, cacheKey digest.Digest) (Result, bool, error) + Link(ctx context.Context, cacheKey digest.Digest, refID string) error } // runOnceCtrl is a simple wrapper around an LRU cache. It's used to ensure that @@ -67,35 +66,30 @@ func (s *runOnceCtrl) hasRun(d digest.Digest, sessionID string) bool { } type simpleSolver struct { - resolveOpFunc ResolveOpFunc - isRunOnceFunc IsRunOnceFunc - commitRefFunc CommitRefFunc - solver *Solver - parallelGuard *parallelGuard - refIDStore *RefIDStore - resultSource ResultSource - cacheKeyManager *cacheKeyManager - runOnceCtrl *runOnceCtrl + resolveOpFunc ResolveOpFunc + isRunOnceFunc IsRunOnceFunc + commitRefFunc CommitRefFunc + solver *Solver + parallelGuard *parallelGuard + resultSource ResultSource + runOnceCtrl *runOnceCtrl } func newSimpleSolver( resolveOpFunc ResolveOpFunc, commitRefFunc CommitRefFunc, solver *Solver, - refIDStore *RefIDStore, resultSource ResultSource, isRunOnceFunc IsRunOnceFunc, ) *simpleSolver { return &simpleSolver{ - cacheKeyManager: newCacheKeyManager(), - parallelGuard: newParallelGuard(parallelGuardWait), - resolveOpFunc: resolveOpFunc, - commitRefFunc: commitRefFunc, - solver: solver, - refIDStore: refIDStore, - resultSource: resultSource, - isRunOnceFunc: isRunOnceFunc, - runOnceCtrl: newRunOnceCtrl(), + parallelGuard: newParallelGuard(parallelGuardWait), + resolveOpFunc: resolveOpFunc, + commitRefFunc: commitRefFunc, + solver: solver, + resultSource: resultSource, + isRunOnceFunc: isRunOnceFunc, + runOnceCtrl: newRunOnceCtrl(), } } @@ -107,13 +101,15 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul var ret Result var expKeys []ExportableCacheKey + runCacheMan := newCacheKeyManager() + for _, d := range digests { vertex, ok := vertices[d] if !ok { return nil, errors.Errorf("digest %s not found", d) } - res, cacheKey, err := s.buildOne(ctx, d, vertex, job, e) + res, cacheKey, err := s.buildOne(ctx, runCacheMan, d, vertex, job, e) if err != nil { return nil, err } @@ -146,7 +142,7 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return NewCachedResult(ret, expKeys), nil } -func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, digest.Digest, error) { +func (s *simpleSolver) buildOne(ctx context.Context, runCacheMan *cacheKeyManager, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, digest.Digest, error) { st := s.state(vertex, job) // Add cache opts to context as they will be accessed by cache retrieval. @@ -158,13 +154,13 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver return nil, "", err } - inputs, err := s.preprocessInputs(ctx, st, vertex, cm.CacheMap, job) + inputs, err := s.preprocessInputs(ctx, runCacheMan, st, vertex, cm.CacheMap, job) if err != nil { notifyError(ctx, st, false, err) return nil, "", err } - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d) + cacheKey, err := runCacheMan.cacheKey(ctx, d) if err != nil { return nil, "", err } @@ -214,7 +210,7 @@ func (s *simpleSolver) buildOne(ctx context.Context, d digest.Digest, vertex Ver } } - err = s.refIDStore.Set(ctx, cacheKey, res.ID()) + err = s.resultSource.Link(ctx, cacheKey, res.ID()) if err != nil { return nil, "", err } @@ -306,7 +302,7 @@ func (s *simpleSolver) exploreVertices(e Edge) ([]digest.Digest, map[digest.Dige return ret, vertices } -func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex Vertex, cm *CacheMap, job *Job) ([]Result, error) { +func (s *simpleSolver) preprocessInputs(ctx context.Context, runCacheMan *cacheKeyManager, st *state, vertex Vertex, cm *CacheMap, job *Job) ([]Result, error) { // This struct is used to reconstruct a cache key from an LLB digest & all // parents using consistent digests that depend on the full dependency chain. scm := simpleCacheMap{ @@ -330,7 +326,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V d := in.Vertex.Digest() // Compute a cache key given the LLB digest value. - cacheKey, err := s.cacheKeyManager.cacheKey(ctx, d) + cacheKey, err := runCacheMan.cacheKey(ctx, d) if err != nil { return nil, err } @@ -390,7 +386,7 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, st *state, vertex V inputs = append(inputs, res) } - s.cacheKeyManager.add(vertex.Digest(), &scm) + runCacheMan.add(vertex.Digest(), &scm) return inputs, nil } @@ -523,76 +519,6 @@ func (f *parallelGuard) acquire(ctx context.Context, d digest.Digest) (<-chan st return ch, closer } -// RefIDStore uses a BoltDB database to store links from computed cache keys to -// worker ref IDs. -type RefIDStore struct { - db *bolt.DB - bucketName string - rootDir string -} - -// NewRefIDStore creates and returns a new store and initializes a BoltDB -// instance in the specified root directory. -func NewRefIDStore(rootDir string) (*RefIDStore, error) { - r := &RefIDStore{ - bucketName: "ids", - rootDir: rootDir, - } - err := r.init() - if err != nil { - return nil, err - } - return r, nil -} - -func (r *RefIDStore) init() error { - db, err := bolt.Open(filepath.Join(r.rootDir, "ids.db"), 0755, nil) - if err != nil { - return err - } - err = db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("ids")) - return err - }) - if err != nil { - return err - } - r.db = db - return nil -} - -// Set a cache key digest to the value of the worker ref ID. -func (r *RefIDStore) Set(ctx context.Context, key digest.Digest, id string) error { - return r.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(r.bucketName)) - return b.Put([]byte(key), []byte(id)) - }) -} - -// Get a worker ref ID given a cache key digest. -func (r *RefIDStore) Get(ctx context.Context, cacheKey digest.Digest) (string, bool, error) { - var id string - err := r.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(r.bucketName)) - id = string(b.Get([]byte(cacheKey))) - return nil - }) - if err != nil { - return "", false, err - } - if id == "" { - return "", false, nil - } - return id, true, nil -} - -func (r *RefIDStore) delete(_ context.Context, key string) error { - return r.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(r.bucketName)) - return b.Delete([]byte(key)) - }) -} - func newDigest(s string) digest.Digest { return digest.NewDigestFromEncoded(digest.SHA256, s) } diff --git a/worker/simple.go b/worker/simple.go index a5a4a4344..17a2c8165 100644 --- a/worker/simple.go +++ b/worker/simple.go @@ -2,34 +2,46 @@ package worker import ( "context" + "path/filepath" "sync" + "time" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) -// RefIDSource allows the caller to translate between a cache key and a worker ref ID. -type RefIDSource interface { - Get(ctx context.Context, cacheKey digest.Digest) (string, bool, error) -} +const refIDPrunePeriod = 30 * time.Minute // WorkerResultSource abstracts the work involved in loading a Result from a // worker using a ref ID. type WorkerResultSource struct { - wc *Controller - ids RefIDSource + wc *Controller + ids *refIDStore + prunePeriod time.Duration } // NewWorkerResultSource creates and returns a new *WorkerResultSource. -func NewWorkerResultSource(wc *Controller, ids RefIDSource) *WorkerResultSource { - return &WorkerResultSource{wc: wc, ids: ids} +func NewWorkerResultSource(wc *Controller, rootDir string) (*WorkerResultSource, error) { + ids, err := newRefIDStore(rootDir) + if err != nil { + return nil, err + } + w := &WorkerResultSource{ + wc: wc, + ids: ids, + prunePeriod: refIDPrunePeriod, + } + go w.pruneLoop(context.Background()) + return w, nil } // Load a cached result from a worker. func (w *WorkerResultSource) Load(ctx context.Context, cacheKey digest.Digest) (solver.Result, bool, error) { - id, ok, err := w.ids.Get(ctx, cacheKey) + fullID, ok, err := w.ids.get(cacheKey) if err != nil { return nil, false, err } @@ -38,7 +50,7 @@ func (w *WorkerResultSource) Load(ctx context.Context, cacheKey digest.Digest) ( return nil, false, nil } - workerID, refID, err := parseWorkerRef(id) + workerID, refID, err := parseWorkerRef(fullID) if err != nil { return nil, false, err } @@ -51,7 +63,7 @@ func (w *WorkerResultSource) Load(ctx context.Context, cacheKey digest.Digest) ( ref, err := worker.LoadRef(ctx, refID, false) if err != nil { if cache.IsNotFound(err) { - bklog.G(ctx).Warnf("could not load ref from worker: %v", err) + bklog.G(ctx).Warnf("worker ref not found: %v", err) return nil, false, nil } return nil, false, err @@ -60,6 +72,84 @@ func (w *WorkerResultSource) Load(ctx context.Context, cacheKey digest.Digest) ( return NewWorkerRefResult(ref, worker), true, nil } +// Link a simple solver cache key to the worker ref. +func (w *WorkerResultSource) Link(ctx context.Context, cacheKey digest.Digest, refID string) error { + return w.ids.set(cacheKey, refID) +} + +// pruneLoop attempts to prune stale red IDs from the BoltDB database every prunePeriod. +func (w *WorkerResultSource) pruneLoop(ctx context.Context) { + tick := time.NewTicker(w.prunePeriod) + for range tick.C { + start := time.Now() + examined, pruned, err := w.prune(ctx) + if err != nil { + bklog.G(ctx).Warnf("failed to prune ref IDs: %v", err) + } else { + bklog.G(ctx).Warnf("examined %d, pruned %d stale ref IDs in %s", examined, pruned, time.Now().Sub(start)) + } + } +} + +// exists determines if the worker ref ID exists. It's important that the ref is +// released after being loaded. +func (w *WorkerResultSource) exists(ctx context.Context, fullID string) (bool, error) { + workerID, refID, err := parseWorkerRef(fullID) + if err != nil { + return false, err + } + + worker, err := w.wc.Get(workerID) + if err != nil { + return false, err + } + + ref, err := worker.LoadRef(ctx, refID, false) + if err != nil { + if cache.IsNotFound(err) { + return false, nil + } + return false, err + } + + ref.Release(ctx) + + return true, nil +} + +// prune ref IDs by identifying and purging all stale IDs. Ref IDs are unique +// and will not be rewritten by a fresh build. Hence, it's safe to delete these +// items once a worker ref has been pruned by BuildKit. +func (w *WorkerResultSource) prune(ctx context.Context) (int, int, error) { + var deleteIDs []digest.Digest + var examined, pruned int + + err := w.ids.walk(func(d digest.Digest, id string) error { + exists, err := w.exists(ctx, id) + if err != nil { + return err + } + examined++ + if !exists { + deleteIDs = append(deleteIDs, d) + } + return nil + }) + if err != nil { + return examined, 0, err + } + + for _, deleteID := range deleteIDs { + err = w.ids.del(deleteID) + if err != nil { + return examined, pruned, err + } + pruned++ + } + + return examined, pruned, nil +} + var _ solver.ResultSource = &WorkerResultSource{} // FinalizeRef is a convenience function that calls Finalize on a Result's @@ -109,6 +199,10 @@ func (w *WorkerRemoteSource) Load(ctx context.Context, cacheKey digest.Digest) ( return NewWorkerRefResult(ref, w.worker), true, nil } +func (c *WorkerRemoteSource) Link(ctx context.Context, cacheKey digest.Digest, refID string) error { + return nil // noop +} + // AddResult adds a solver.Remote source for the given cache key. func (w *WorkerRemoteSource) AddResult(ctx context.Context, cacheKey digest.Digest, remote *solver.Remote) { w.mu.Lock() @@ -142,4 +236,129 @@ func (c *CombinedResultSource) Load(ctx context.Context, cacheKey digest.Digest) return nil, false, nil } +// Link a cache key to a ref ID. Only used by the worker result source. +func (c *CombinedResultSource) Link(ctx context.Context, cacheKey digest.Digest, refID string) error { + for _, source := range c.sources { + err := source.Link(ctx, cacheKey, refID) + if err != nil { + return nil + } + } + return nil +} + var _ solver.ResultSource = &CombinedResultSource{} + +// refIDStore uses a BoltDB database to store links from computed cache keys to +// worker ref IDs. +type refIDStore struct { + db *bolt.DB + rootDir string + bucket string + prunePeriod time.Duration +} + +// newRefIDStore creates and returns a new store and initializes a BoltDB +// instance in the specified root directory. +func newRefIDStore(rootDir string) (*refIDStore, error) { + r := &refIDStore{ + bucket: "ids", + rootDir: rootDir, + prunePeriod: refIDPrunePeriod, + } + err := r.init() + if err != nil { + return nil, err + } + return r, nil +} + +func (r *refIDStore) init() error { + db, err := bolt.Open(filepath.Join(r.rootDir, "ids.db"), 0755, nil) + if err != nil { + return err + } + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(r.bucket)) + return err + }) + r.db = db + return nil +} + +// set a cache key digest to the value of the worker ref ID. It also sets the +// access time for the key. +func (r *refIDStore) set(cacheKey digest.Digest, id string) error { + err := r.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucket)) + return b.Put([]byte(cacheKey), []byte(id)) + }) + if err != nil { + return errors.Wrap(err, "failed to set ref ID") + } + return nil +} + +// get a worker ref ID given a cache key digest. It also sets the +// access time for the key. +func (r *refIDStore) get(cacheKey digest.Digest) (string, bool, error) { + var id string + err := r.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucket)) + id = string(b.Get([]byte(cacheKey))) + return nil + }) + if err != nil { + return "", false, errors.Wrap(err, "failed to load ref ID") + } + if id == "" { + return "", false, nil + } + return id, true, nil +} + +// del removes a single cache key from the ref ID database. +func (r *refIDStore) del(cacheKey digest.Digest) error { + err := r.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucket)) + return b.Delete([]byte(cacheKey)) + }) + if err != nil { + return errors.Wrap(err, "failed to delete key") + } + return nil +} + +// walk all items in the database using a callback function. +func (r *refIDStore) walk(fn func(digest.Digest, string) error) error { + + all := map[digest.Digest]string{} + + err := r.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(r.bucket)) + err := b.ForEach(func(k, v []byte) error { + d := digest.Digest(string(k)) + all[d] = string(v) + return nil + }) + if err != nil { + return err + } + return nil + }) + if err != nil { + return errors.Wrap(err, "failed to iterate ref IDs") + } + + // The callback needs to be invoked outside of the BoltDB View + // transaction. Update is an option, but it's much slower. ForEach cannot + // modify results inside the loop. + for d, id := range all { + err = fn(d, id) + if err != nil { + return err + } + } + + return nil +} From fca848559c8771cb3915fafd92b6fbd58f5e185a Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 24 May 2024 14:11:16 -0700 Subject: [PATCH 53/54] Commit all refs aside from "run once" group --- solver/simple.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index 752ba3d98..d7050b61f 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -134,11 +134,6 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul }) } - err := s.commitRefFunc(ctx, ret) - if err != nil { - return nil, err - } - return NewCachedResult(ret, expKeys), nil } @@ -210,6 +205,15 @@ func (s *simpleSolver) buildOne(ctx context.Context, runCacheMan *cacheKeyManage } } + // Some operations need to be left in a mutable state. All others need to be + // committed in order to be cached and loaded correctly. + if !isRunOnce { + err = s.commitRefFunc(ctx, res) + if err != nil { + return nil, "", err + } + } + err = s.resultSource.Link(ctx, cacheKey, res.ID()) if err != nil { return nil, "", err @@ -445,14 +449,8 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d digest.Digest, io.WriteString(h, c.salt) } - for _, in := range c.inputs { - err := m.cacheKeyRecurse(ctx, in, h) - if err != nil { - return err - } - } - io.WriteString(h, c.digest.String()) + for _, dep := range c.deps { if dep.selector != "" { io.WriteString(h, dep.selector.String()) @@ -462,6 +460,13 @@ func (m *cacheKeyManager) cacheKeyRecurse(ctx context.Context, d digest.Digest, } } + for _, in := range c.inputs { + err := m.cacheKeyRecurse(ctx, in, h) + if err != nil { + return err + } + } + return nil } From 88ecf5d6f17aa02643b80697f5251a2b2c1538e9 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 27 May 2024 11:39:08 -0700 Subject: [PATCH 54/54] Perf improvements to slow cache & release --- solver/simple.go | 119 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 86 insertions(+), 33 deletions(-) diff --git a/solver/simple.go b/solver/simple.go index d7050b61f..d8ec1c940 100644 --- a/solver/simple.go +++ b/solver/simple.go @@ -19,6 +19,7 @@ import ( const ( runOnceLRUSize = 2000 + slowCacheLRUSize = 5000 parallelGuardWait = time.Millisecond * 100 ) @@ -65,14 +66,46 @@ func (s *runOnceCtrl) hasRun(d digest.Digest, sessionID string) bool { return ret } +type slowCacheStore struct { + lru *simplelru.LRU + mu sync.Mutex +} + +func newSlowCacheStore() *slowCacheStore { + lru, _ := simplelru.NewLRU(slowCacheLRUSize, nil) // Error impossible on positive first argument. + return &slowCacheStore{lru: lru} +} + +func (s *slowCacheStore) get(cacheKey digest.Digest, refID string) (digest.Digest, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + key := fmt.Sprintf("%s:%s", cacheKey, refID) + v, ok := s.lru.Get(key) + if !ok { + return "", false + } + + return v.(digest.Digest), true +} + +func (s *slowCacheStore) set(cacheKey digest.Digest, refID string, slow digest.Digest) { + s.mu.Lock() + defer s.mu.Unlock() + + key := fmt.Sprintf("%s:%s", cacheKey, refID) + s.lru.Add(key, slow) +} + type simpleSolver struct { - resolveOpFunc ResolveOpFunc - isRunOnceFunc IsRunOnceFunc - commitRefFunc CommitRefFunc - solver *Solver - parallelGuard *parallelGuard - resultSource ResultSource - runOnceCtrl *runOnceCtrl + resolveOpFunc ResolveOpFunc + isRunOnceFunc IsRunOnceFunc + commitRefFunc CommitRefFunc + solver *Solver + parallelGuard *parallelGuard + resultSource ResultSource + runOnceCtrl *runOnceCtrl + slowCacheStore *slowCacheStore } func newSimpleSolver( @@ -83,13 +116,14 @@ func newSimpleSolver( isRunOnceFunc IsRunOnceFunc, ) *simpleSolver { return &simpleSolver{ - parallelGuard: newParallelGuard(parallelGuardWait), - resolveOpFunc: resolveOpFunc, - commitRefFunc: commitRefFunc, - solver: solver, - resultSource: resultSource, - isRunOnceFunc: isRunOnceFunc, - runOnceCtrl: newRunOnceCtrl(), + parallelGuard: newParallelGuard(parallelGuardWait), + resolveOpFunc: resolveOpFunc, + commitRefFunc: commitRefFunc, + solver: solver, + resultSource: resultSource, + isRunOnceFunc: isRunOnceFunc, + runOnceCtrl: newRunOnceCtrl(), + slowCacheStore: newSlowCacheStore(), } } @@ -103,6 +137,8 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul runCacheMan := newCacheKeyManager() + closers := []func(context.Context) error{} + for _, d := range digests { vertex, ok := vertices[d] if !ok { @@ -114,17 +150,14 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul return nil, err } - // Release previous result as this is not the final return value. - if ret != nil { - err := ret.Release(ctx) - if err != nil { - return nil, err - } - } + closers = append(closers, func(ctx context.Context) error { + return res.Release(ctx) + }) ret = res - // Hijack the CacheKey type in order to export a reference from the new cache key to the ref ID. + // Hijack the CacheKey type in order to export a reference from the new + // cache key to the ref ID. expKeys = append(expKeys, ExportableCacheKey{ CacheKey: &CacheKey{ ID: res.ID(), @@ -134,10 +167,22 @@ func (s *simpleSolver) build(ctx context.Context, job *Job, e Edge) (CachedResul }) } + // Defer releasing of results until this build has finished to limit + // performance impact. + go func() { + ctx := context.Background() + for i := len(closers) - 1; i >= 0; i-- { + if err := closers[i](ctx); err != nil { + bklog.G(ctx).Warnf("failed to release: %v", err) + } + } + }() + return NewCachedResult(ret, expKeys), nil } func (s *simpleSolver) buildOne(ctx context.Context, runCacheMan *cacheKeyManager, d digest.Digest, vertex Vertex, job *Job, e Edge) (Result, digest.Digest, error) { + st := s.state(vertex, job) // Add cache opts to context as they will be accessed by cache retrieval. @@ -362,25 +407,33 @@ func (s *simpleSolver) preprocessInputs(ctx context.Context, runCacheMan *cacheK // ComputeDigestFunc will usually checksum files. This is then used as // part of the cache key to ensure it's consistent & distinct for this - // operation. + // operation. The key is then cached based on the key calculated from + // all ancestors & the result ID. if dep.ComputeDigestFunc != nil { - compDigest, err := dep.ComputeDigestFunc(ctx, res, st) - if err != nil { - bklog.G(ctx).Warnf("failed to compute digest: %v", err) - return nil, err + cachedSlowKey, ok := s.slowCacheStore.get(cacheKey, res.ID()) + if ok { + scm.deps[i].computed = cachedSlowKey } else { - scm.deps[i].computed = compDigest + slowKey, err := dep.ComputeDigestFunc(ctx, res, st) + if err != nil { + bklog.G(ctx).Warnf("failed to compute digest: %v", err) + return nil, err + } else { + scm.deps[i].computed = slowKey + s.slowCacheStore.set(cacheKey, res.ID(), slowKey) + } } } // The result can be released now that the preprocess & slow cache // digest functions have been run. This is crucial as failing to do so // will lead to full file copying from previously executed source - // operations. - err = res.Release(ctx) - if err != nil { - return nil, err - } + // operations. Releasing can be slow, so we run these concurrently. + go func() { + if err := res.Release(ctx); err != nil { + bklog.G(ctx).Warnf("failed to release result: %v", err) + } + }() // Add input references to the struct as to link dependencies. scm.inputs[i] = in.Vertex.Digest()