From 44ac44907312b7222349e593491f91c849721898 Mon Sep 17 00:00:00 2001 From: HongboDu-at Date: Tue, 16 Sep 2025 22:17:44 -0700 Subject: [PATCH 1/5] native build directories hardlinking file fetcher de-duplicates in-flight download requests --- pkg/cas/hardlinking_file_fetcher.go | 156 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 47 deletions(-) diff --git a/pkg/cas/hardlinking_file_fetcher.go b/pkg/cas/hardlinking_file_fetcher.go index e98a745d..66b3ad3a 100644 --- a/pkg/cas/hardlinking_file_fetcher.go +++ b/pkg/cas/hardlinking_file_fetcher.go @@ -26,6 +26,14 @@ type hardlinkingFileFetcher struct { evictionLock sync.Mutex evictionSet eviction.Set[string] + + downloadsLock sync.Mutex + downloads map[string]*download +} + +type download struct { + wait chan struct{} + err error } // NewHardlinkingFileFetcher is an adapter for FileFetcher that stores @@ -43,6 +51,8 @@ func NewHardlinkingFileFetcher(base FileFetcher, cacheDirectory filesystem.Direc filesSize: map[string]int64{}, evictionSet: evictionSet, + + downloads: map[string]*download{}, } } @@ -71,59 +81,111 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest } // If the file is present in the cache, hardlink it to the destination. - wasMissing := false - ff.filesLock.RLock() - if _, ok := ff.filesSize[key]; ok { - ff.evictionLock.Lock() - ff.evictionSet.Touch(key) - ff.evictionLock.Unlock() - - if err := ff.cacheDirectory.Link(path.MustNewComponent(key), directory, name); err == nil { - // Successfully hardlinked the file to its destination. - ff.filesLock.RUnlock() - return nil - } else if !os.IsNotExist(err) { - ff.filesLock.RUnlock() - return util.StatusWrapfWithCode(err, codes.Internal, "Failed to create hardlink to cached file %#v", key) - } + wasMissing, err := ff.tryLinkFromCache(key, directory, name) + if err == nil { + return nil + } else if !wasMissing { + return err + } - // The file was part of the cache, even though it did not - // exist on disk. Some other process may have tampered - // with the cache directory's contents. - wasMissing = true + // A download is required. Let's see if one is already in progress. + ff.downloadsLock.Lock() + d, ok := ff.downloads[key] + if ok { + // A download is already in progress. Wait for it to finish. + ff.downloadsLock.Unlock() + select { + case <-d.wait: + if d.err != nil { + return d.err + } + // The download should have placed the file in the cache. + // Try linking from the cache one more time. + wasMissingAfterWait, errAfterWait := ff.tryLinkFromCache(key, directory, name) + if errAfterWait == nil { + return nil + } else if wasMissingAfterWait { + return util.StatusWrapfWithCode(errAfterWait, codes.Internal, "Failed to link from cache for %#v after download", key) + } + return errAfterWait + case <-ctx.Done(): + return util.StatusFromContext(ctx) + } } - ff.filesLock.RUnlock() - // Download the file at the intended location. - if err := ff.base.GetFile(ctx, blobDigest, directory, name, isExecutable); err != nil { - return err + // Start a new download. + d = &download{wait: make(chan struct{})} + ff.downloads[key] = d + ff.downloadsLock.Unlock() + + downloadErr := ff.base.GetFile(ctx, blobDigest, directory, name, isExecutable) + if downloadErr == nil { + // The file was downloaded successfully. Place it into the + // cache, so that successive calls may use it. + ff.filesLock.Lock() + if _, ok := ff.filesSize[key]; !ok { + ff.evictionLock.Lock() + + // Remove old files from the cache if necessary. + sizeBytes := blobDigest.GetSizeBytes() + if err := ff.makeSpace(sizeBytes); err != nil { + downloadErr = err + } else { + // Hardlink the file into the cache. + if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { + downloadErr = util.StatusWrapfWithCode(err, codes.Internal, "Failed to add cached file %#v", key) + } else { + ff.evictionSet.Insert(key) + ff.filesSize[key] = sizeBytes + ff.filesTotalSize += sizeBytes + } + } + ff.evictionLock.Unlock() + } else if wasMissing { + // The file was already part of our bookkeeping, + // but was missing on disk. Repair this by adding + // a link to the newly downloaded file. + if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { + downloadErr = util.StatusWrapfWithCode(err, codes.Internal, "Failed to repair cached file %#v", key) + } + } + ff.filesLock.Unlock() } - ff.filesLock.Lock() - defer ff.filesLock.Unlock() - if _, ok := ff.filesSize[key]; !ok { - ff.evictionLock.Lock() - defer ff.evictionLock.Unlock() + // Unblock waiters. + ff.downloadsLock.Lock() + d.err = downloadErr + delete(ff.downloads, key) + ff.downloadsLock.Unlock() + close(d.wait) - // Remove old files from the cache if necessary. - sizeBytes := blobDigest.GetSizeBytes() - if err := ff.makeSpace(sizeBytes); err != nil { - return err - } + return downloadErr +} - // Hardlink the file into the cache. - if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { - return util.StatusWrapfWithCode(err, codes.Internal, "Failed to add cached file %#v", key) - } - ff.evictionSet.Insert(key) - ff.filesSize[key] = sizeBytes - ff.filesTotalSize += sizeBytes - } else if wasMissing { - // Even though the file is part of our bookkeeping, we - // observed it didn't exist. Repair this inconsistency. - if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { - return util.StatusWrapfWithCode(err, codes.Internal, "Failed to repair cached file %#v", key) - } +// tryLinkFromCache attempts to create a hardlink from the cache to a +// file in the build directory. The first return value is whether the +// file was present in the cache's bookkeeping, but missing on disk. +func (ff *hardlinkingFileFetcher) tryLinkFromCache(key string, directory filesystem.Directory, name path.Component) (bool, error) { + ff.filesLock.RLock() + _, ok := ff.filesSize[key] + ff.filesLock.RUnlock() + if !ok { + return true, os.ErrNotExist } - return nil + + ff.evictionLock.Lock() + ff.evictionSet.Touch(key) + ff.evictionLock.Unlock() + + if err := ff.cacheDirectory.Link(path.MustNewComponent(key), directory, name); err == nil { + // Successfully hardlinked the file to its destination. + return false, nil + } else if !os.IsNotExist(err) { + return false, util.StatusWrapfWithCode(err, codes.Internal, "Failed to create hardlink to cached file %#v", key) + } + + // The file was part of the cache, even though it did not + // exist on disk. Some other process may have tampered with + // the cache directory's contents. + return true, os.ErrNotExist } From 5c7e9653645027710c1b016213e3ff3cae9fe6d1 Mon Sep 17 00:00:00 2001 From: HongboDu-at Date: Wed, 7 Jan 2026 14:52:25 -0800 Subject: [PATCH 2/5] Callers retry download for context cancellation; fix one possible redundant download --- pkg/cas/hardlinking_file_fetcher.go | 31 ++++++++++++++++++++++-- pkg/cas/hardlinking_file_fetcher_test.go | 6 +++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/cas/hardlinking_file_fetcher.go b/pkg/cas/hardlinking_file_fetcher.go index 66b3ad3a..2e4b2bc2 100644 --- a/pkg/cas/hardlinking_file_fetcher.go +++ b/pkg/cas/hardlinking_file_fetcher.go @@ -2,6 +2,7 @@ package cas import ( "context" + "errors" "os" "sync" @@ -97,6 +98,11 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest select { case <-d.wait: if d.err != nil { + // If the download failed due to context cancellation, + // and our context is still valid, retry with our context. + if ctx.Err() == nil && errors.Is(d.err, context.Canceled) { + return ff.GetFile(ctx, blobDigest, directory, name, isExecutable) + } return d.err } // The download should have placed the file in the cache. @@ -118,6 +124,26 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest ff.downloads[key] = d ff.downloadsLock.Unlock() + // Check cache again in case another download completed between our initial + // tryLinkFromCache() call and acquiring the download lock. + wasMissing, err = ff.tryLinkFromCache(key, directory, name) + if err == nil { + // File appeared in cache, no download needed. + ff.downloadsLock.Lock() + delete(ff.downloads, key) + ff.downloadsLock.Unlock() + close(d.wait) + return nil + } else if !wasMissing { + // tryLinkFromCache had a real error (not just missing). + ff.downloadsLock.Lock() + d.err = err + delete(ff.downloads, key) + ff.downloadsLock.Unlock() + close(d.wait) + return err + } + downloadErr := ff.base.GetFile(ctx, blobDigest, directory, name, isExecutable) if downloadErr == nil { // The file was downloaded successfully. Place it into the @@ -163,8 +189,9 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest } // tryLinkFromCache attempts to create a hardlink from the cache to a -// file in the build directory. The first return value is whether the -// file was present in the cache's bookkeeping, but missing on disk. +// file in the build directory. The first return value indicates whether +// a download is needed: true if the file is not in the cache bookkeeping, +// or if it was in bookkeeping but missing on disk. func (ff *hardlinkingFileFetcher) tryLinkFromCache(key string, directory filesystem.Directory, name path.Component) (bool, error) { ff.filesLock.RLock() _, ok := ff.filesSize[key] diff --git a/pkg/cas/hardlinking_file_fetcher_test.go b/pkg/cas/hardlinking_file_fetcher_test.go index a2ab6704..953c2656 100644 --- a/pkg/cas/hardlinking_file_fetcher_test.go +++ b/pkg/cas/hardlinking_file_fetcher_test.go @@ -74,6 +74,8 @@ func TestHardlinkingFileFetcher(t *testing.T) { // Recover from the case where the cache directory gets cleaned // up by another process. If hardlinking returns ENOENT, we // should fall back to downloading and reinserting the file. + cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). + Return(syscall.ENOENT) cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). Return(syscall.ENOENT) baseFileFetcher.EXPECT().GetFile(ctx, blobDigest1, buildDirectory, path.MustNewComponent("hello.txt"), false) @@ -84,6 +86,8 @@ func TestHardlinkingFileFetcher(t *testing.T) { // The above may happen in multiple threads at the same time. // EEXIST errors should be ignored in that case. + cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). + Return(syscall.ENOENT) cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). Return(syscall.ENOENT) baseFileFetcher.EXPECT().GetFile(ctx, blobDigest1, buildDirectory, path.MustNewComponent("hello.txt"), false) @@ -94,6 +98,8 @@ func TestHardlinkingFileFetcher(t *testing.T) { fileFetcher.GetFile(ctx, blobDigest1, buildDirectory, path.MustNewComponent("hello.txt"), false)) // Errors other than EEXIST should be propagated as usual. + cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). + Return(syscall.ENOENT) cacheDirectory.EXPECT().Link(path.MustNewComponent("3-8b1a9953c4611296a827abf8c47804d7-5-x"), buildDirectory, path.MustNewComponent("hello.txt")). Return(syscall.ENOENT) baseFileFetcher.EXPECT().GetFile(ctx, blobDigest1, buildDirectory, path.MustNewComponent("hello.txt"), false) From cc74acf04738f5dadd08670d8569cde4f9cb5be8 Mon Sep 17 00:00:00 2001 From: HongboDu-at Date: Thu, 8 Jan 2026 16:48:40 -0800 Subject: [PATCH 3/5] use downloads map[string]<-chan struct{} and defer --- pkg/cas/hardlinking_file_fetcher.go | 65 +++++++++-------------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/pkg/cas/hardlinking_file_fetcher.go b/pkg/cas/hardlinking_file_fetcher.go index 2e4b2bc2..8f0cb4c2 100644 --- a/pkg/cas/hardlinking_file_fetcher.go +++ b/pkg/cas/hardlinking_file_fetcher.go @@ -2,7 +2,6 @@ package cas import ( "context" - "errors" "os" "sync" @@ -29,12 +28,7 @@ type hardlinkingFileFetcher struct { evictionSet eviction.Set[string] downloadsLock sync.Mutex - downloads map[string]*download -} - -type download struct { - wait chan struct{} - err error + downloads map[string]<-chan struct{} } // NewHardlinkingFileFetcher is an adapter for FileFetcher that stores @@ -53,7 +47,7 @@ func NewHardlinkingFileFetcher(base FileFetcher, cacheDirectory filesystem.Direc evictionSet: evictionSet, - downloads: map[string]*download{}, + downloads: map[string]<-chan struct{}{}, } } @@ -91,56 +85,44 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest // A download is required. Let's see if one is already in progress. ff.downloadsLock.Lock() - d, ok := ff.downloads[key] + wait, ok := ff.downloads[key] if ok { // A download is already in progress. Wait for it to finish. ff.downloadsLock.Unlock() select { - case <-d.wait: - if d.err != nil { - // If the download failed due to context cancellation, - // and our context is still valid, retry with our context. - if ctx.Err() == nil && errors.Is(d.err, context.Canceled) { - return ff.GetFile(ctx, blobDigest, directory, name, isExecutable) - } - return d.err - } - // The download should have placed the file in the cache. - // Try linking from the cache one more time. - wasMissingAfterWait, errAfterWait := ff.tryLinkFromCache(key, directory, name) - if errAfterWait == nil { + case <-wait: + // Try linking from cache. If missing (download failed or + // other issue), retry GetFile which will attempt a new download. + wasMissing, err := ff.tryLinkFromCache(key, directory, name) + if err == nil { return nil - } else if wasMissingAfterWait { - return util.StatusWrapfWithCode(errAfterWait, codes.Internal, "Failed to link from cache for %#v after download", key) + } else if wasMissing { + return ff.GetFile(ctx, blobDigest, directory, name, isExecutable) } - return errAfterWait + return err case <-ctx.Done(): return util.StatusFromContext(ctx) } } // Start a new download. - d = &download{wait: make(chan struct{})} - ff.downloads[key] = d + newWait := make(chan struct{}) + ff.downloads[key] = newWait ff.downloadsLock.Unlock() + defer func() { + ff.downloadsLock.Lock() + delete(ff.downloads, key) + ff.downloadsLock.Unlock() + close(newWait) + }() + // Check cache again in case another download completed between our initial // tryLinkFromCache() call and acquiring the download lock. wasMissing, err = ff.tryLinkFromCache(key, directory, name) if err == nil { - // File appeared in cache, no download needed. - ff.downloadsLock.Lock() - delete(ff.downloads, key) - ff.downloadsLock.Unlock() - close(d.wait) return nil } else if !wasMissing { - // tryLinkFromCache had a real error (not just missing). - ff.downloadsLock.Lock() - d.err = err - delete(ff.downloads, key) - ff.downloadsLock.Unlock() - close(d.wait) return err } @@ -178,13 +160,6 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest ff.filesLock.Unlock() } - // Unblock waiters. - ff.downloadsLock.Lock() - d.err = downloadErr - delete(ff.downloads, key) - ff.downloadsLock.Unlock() - close(d.wait) - return downloadErr } From e12099a49b1a37155bd2ad7435c4c85fa7501720 Mon Sep 17 00:00:00 2001 From: HongboDu-at Date: Mon, 12 Jan 2026 13:45:10 -0800 Subject: [PATCH 4/5] call call os.IsNotExist(err); For loop GetFile() --- pkg/cas/hardlinking_file_fetcher.go | 74 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/pkg/cas/hardlinking_file_fetcher.go b/pkg/cas/hardlinking_file_fetcher.go index 8f0cb4c2..a47e1347 100644 --- a/pkg/cas/hardlinking_file_fetcher.go +++ b/pkg/cas/hardlinking_file_fetcher.go @@ -75,37 +75,34 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest key += "-x" } - // If the file is present in the cache, hardlink it to the destination. - wasMissing, err := ff.tryLinkFromCache(key, directory, name) - if err == nil { - return nil - } else if !wasMissing { - return err - } + for { + // If the file is present in the cache, hardlink it to the destination. + if err := ff.tryLinkFromCache(key, directory, name); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } - // A download is required. Let's see if one is already in progress. - ff.downloadsLock.Lock() - wait, ok := ff.downloads[key] - if ok { - // A download is already in progress. Wait for it to finish. - ff.downloadsLock.Unlock() - select { - case <-wait: - // Try linking from cache. If missing (download failed or - // other issue), retry GetFile which will attempt a new download. - wasMissing, err := ff.tryLinkFromCache(key, directory, name) - if err == nil { - return nil - } else if wasMissing { - return ff.GetFile(ctx, blobDigest, directory, name, isExecutable) + // A download is required. Let's see if one is already in progress. + ff.downloadsLock.Lock() + wait, ok := ff.downloads[key] + if ok { + // A download is already in progress. Wait for it to finish. + ff.downloadsLock.Unlock() + select { + case <-wait: + // Download finished. Loop back to try linking from + // cache. If missing (download failed or other issue), + // we'll attempt a new download. + continue + case <-ctx.Done(): + return util.StatusFromContext(ctx) } - return err - case <-ctx.Done(): - return util.StatusFromContext(ctx) } - } - // Start a new download. + // Start a new download. + break + } newWait := make(chan struct{}) ff.downloads[key] = newWait ff.downloadsLock.Unlock() @@ -119,10 +116,9 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest // Check cache again in case another download completed between our initial // tryLinkFromCache() call and acquiring the download lock. - wasMissing, err = ff.tryLinkFromCache(key, directory, name) - if err == nil { + if err := ff.tryLinkFromCache(key, directory, name); err == nil { return nil - } else if !wasMissing { + } else if !os.IsNotExist(err) { return err } @@ -149,7 +145,7 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest } } ff.evictionLock.Unlock() - } else if wasMissing { + } else { // The file was already part of our bookkeeping, // but was missing on disk. Repair this by adding // a link to the newly downloaded file. @@ -164,15 +160,15 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest } // tryLinkFromCache attempts to create a hardlink from the cache to a -// file in the build directory. The first return value indicates whether -// a download is needed: true if the file is not in the cache bookkeeping, -// or if it was in bookkeeping but missing on disk. -func (ff *hardlinkingFileFetcher) tryLinkFromCache(key string, directory filesystem.Directory, name path.Component) (bool, error) { +// file in the build directory. It returns os.ErrNotExist if the file +// is not in the cache bookkeeping, or if it was in bookkeeping but +// missing on disk. +func (ff *hardlinkingFileFetcher) tryLinkFromCache(key string, directory filesystem.Directory, name path.Component) error { ff.filesLock.RLock() _, ok := ff.filesSize[key] ff.filesLock.RUnlock() if !ok { - return true, os.ErrNotExist + return os.ErrNotExist } ff.evictionLock.Lock() @@ -181,13 +177,13 @@ func (ff *hardlinkingFileFetcher) tryLinkFromCache(key string, directory filesys if err := ff.cacheDirectory.Link(path.MustNewComponent(key), directory, name); err == nil { // Successfully hardlinked the file to its destination. - return false, nil + return nil } else if !os.IsNotExist(err) { - return false, util.StatusWrapfWithCode(err, codes.Internal, "Failed to create hardlink to cached file %#v", key) + return util.StatusWrapfWithCode(err, codes.Internal, "Failed to create hardlink to cached file %#v", key) } // The file was part of the cache, even though it did not // exist on disk. Some other process may have tampered with // the cache directory's contents. - return true, os.ErrNotExist + return os.ErrNotExist } From deb36dfd4f5d1cb2f6a6bfa51d94ba9cfab195d6 Mon Sep 17 00:00:00 2001 From: HongboDu-at Date: Tue, 13 Jan 2026 11:56:45 -0800 Subject: [PATCH 5/5] revert the cache insertion part --- pkg/cas/hardlinking_file_fetcher.go | 63 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/pkg/cas/hardlinking_file_fetcher.go b/pkg/cas/hardlinking_file_fetcher.go index a47e1347..f548fd81 100644 --- a/pkg/cas/hardlinking_file_fetcher.go +++ b/pkg/cas/hardlinking_file_fetcher.go @@ -122,41 +122,38 @@ func (ff *hardlinkingFileFetcher) GetFile(ctx context.Context, blobDigest digest return err } - downloadErr := ff.base.GetFile(ctx, blobDigest, directory, name, isExecutable) - if downloadErr == nil { - // The file was downloaded successfully. Place it into the - // cache, so that successive calls may use it. - ff.filesLock.Lock() - if _, ok := ff.filesSize[key]; !ok { - ff.evictionLock.Lock() - - // Remove old files from the cache if necessary. - sizeBytes := blobDigest.GetSizeBytes() - if err := ff.makeSpace(sizeBytes); err != nil { - downloadErr = err - } else { - // Hardlink the file into the cache. - if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { - downloadErr = util.StatusWrapfWithCode(err, codes.Internal, "Failed to add cached file %#v", key) - } else { - ff.evictionSet.Insert(key) - ff.filesSize[key] = sizeBytes - ff.filesTotalSize += sizeBytes - } - } - ff.evictionLock.Unlock() - } else { - // The file was already part of our bookkeeping, - // but was missing on disk. Repair this by adding - // a link to the newly downloaded file. - if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { - downloadErr = util.StatusWrapfWithCode(err, codes.Internal, "Failed to repair cached file %#v", key) - } - } - ff.filesLock.Unlock() + // Download the file at the intended location. + if err := ff.base.GetFile(ctx, blobDigest, directory, name, isExecutable); err != nil { + return err } - return downloadErr + ff.filesLock.Lock() + defer ff.filesLock.Unlock() + if _, ok := ff.filesSize[key]; !ok { + ff.evictionLock.Lock() + defer ff.evictionLock.Unlock() + + // Remove old files from the cache if necessary. + sizeBytes := blobDigest.GetSizeBytes() + if err := ff.makeSpace(sizeBytes); err != nil { + return err + } + + // Hardlink the file into the cache. + if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { + return util.StatusWrapfWithCode(err, codes.Internal, "Failed to add cached file %#v", key) + } + ff.evictionSet.Insert(key) + ff.filesSize[key] = sizeBytes + ff.filesTotalSize += sizeBytes + } else { + // Even though the file is part of our bookkeeping, we + // observed it didn't exist. Repair this inconsistency. + if err := directory.Link(name, ff.cacheDirectory, path.MustNewComponent(key)); err != nil && !os.IsExist(err) { + return util.StatusWrapfWithCode(err, codes.Internal, "Failed to repair cached file %#v", key) + } + } + return nil } // tryLinkFromCache attempts to create a hardlink from the cache to a