Skip to content

Commit 13f4cf1

Browse files
0x416e746f6e0x416e746f6e
committed
feat: handle attested images
1 parent 9c39fec commit 13f4cf1

File tree

5 files changed

+195
-132
lines changed

5 files changed

+195
-132
lines changed

‎config/destination.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func (cfg *Destination) Validate() error {
7575
return utils.FlattenErrors(errs)
7676
}
7777

78-
func (cfg *Destination) HasPlatform(p string) bool {
79-
_, has := cfg.platforms[p]
78+
func (cfg *Destination) HasPlatform(p *cr.Platform) bool {
79+
_, has := cfg.platforms[p.String()] // TODO: enable globbing and matching
8080
return has
8181
}

‎job/sync_container_registry_package.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,13 @@ func (j *SyncContainerRegistryPackage) GetDestinations() []*config.Destination {
5757
}
5858

5959
func (j *SyncContainerRegistryPackage) GetDestinationReference(dst *config.Destination) string {
60-
if *j.Package.PackageVersion.ContainerMetadata.Tag.Name == "" {
61-
return dst.Path + "/" + dst.Package + "@" + *j.Package.PackageVersion.ContainerMetadata.Tag.Digest
60+
tag := *j.Package.PackageVersion.ContainerMetadata.Tag.Name
61+
if tag == "" {
62+
tag = strings.ReplaceAll(
63+
*j.Package.PackageVersion.ContainerMetadata.Tag.Digest, ":", "-",
64+
)
6265
}
63-
return dst.Path + "/" + dst.Package + ":" + *j.Package.PackageVersion.ContainerMetadata.Tag.Name
66+
return dst.Path + "/" + dst.Package + ":" + tag
6467
}
6568

6669
func (j *SyncContainerRegistryPackage) GetDigest() string {

‎server/download_github_container.go‎

Lines changed: 61 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"net/http"
99
"os"
1010
"path/filepath"
11+
"strings"
1112
"time"
1213

1314
"github.com/flashbots/gh-artifacts-sync/job"
@@ -30,9 +31,9 @@ func (s *Server) downloadGithubContainer(
3031

3132
var downloadsDir string
3233
{ // create container downloads dir
33-
downloadsDir = filepath.Join(
34-
s.cfg.Dir.Downloads, j.GetRepoOwner(), j.GetRepo(), "containers", j.GetPackageName(), j.GetTag(),
35-
)
34+
downloadsDir = strings.ReplaceAll(filepath.Join(
35+
s.cfg.Dir.Downloads, j.GetRepoOwner(), j.GetRepo(), "containers", j.GetPackageName(), j.GetDigest(),
36+
), ":", "-")
3637
if err := os.MkdirAll(downloadsDir, 0750); err != nil {
3738
return "", fmt.Errorf("failed to create container download directory: %s: %w",
3839
downloadsDir, err,
@@ -89,7 +90,7 @@ func (s *Server) downloadGithubContainer(
8990
desc = _desc
9091
}
9192

92-
var images = make(map[string]cr.Image)
93+
var images = make(map[string][]cr.Image)
9394
{ // get images
9495
switch {
9596
case desc.MediaType.IsImage():
@@ -98,16 +99,21 @@ func (s *Server) downloadGithubContainer(
9899
return "", fmt.Errorf("failed to retrieve container image: %w", err)
99100
}
100101

101-
platform := ""
102+
platform := "unknown/unknown"
102103
if desc.Platform != nil {
103104
platform = desc.Platform.String()
104105
}
105-
106-
if _, exists := images[platform]; exists {
107-
return "", fmt.Errorf("invalid container image: duplicate platform: %s", platform)
106+
if _, exists := images[platform]; !exists {
107+
images[platform] = make([]cr.Image, 0)
108108
}
109109

110-
images[platform] = image
110+
images[platform] = append(images[platform], image)
111+
112+
l.Debug("Downloaded a manifest",
113+
zap.String("digest", desc.Digest.String()),
114+
zap.String("platform", platform),
115+
zap.Any("annotations", desc.Annotations),
116+
)
111117

112118
case desc.MediaType.IsIndex():
113119
index, err := crremote.Index(ref, crremote.WithAuth(auth))
@@ -122,24 +128,39 @@ func (s *Server) downloadGithubContainer(
122128
)
123129
}
124130

131+
l.Debug("Downloaded an index",
132+
zap.String("digest", desc.Digest.String()),
133+
zap.String("reference", ref.Name()),
134+
zap.Any("annotations", indexManifest.Annotations),
135+
)
136+
125137
for _, desc := range indexManifest.Manifests {
126-
if !desc.MediaType.IsImage() || desc.Platform == nil {
138+
if !desc.MediaType.IsImage() {
127139
continue
128140
}
129141

130-
platform := desc.Platform.String()
131-
if _, exists := images[platform]; exists {
132-
return "", fmt.Errorf("invalid container image: duplicate platform: %s", platform)
133-
}
134-
135142
image, err := index.Image(desc.Digest)
136143
if err != nil {
137144
return "", fmt.Errorf("failed to get image from an index: %s: %s: %w",
138145
j.GetPackageUrl(), desc.Digest, err,
139146
)
140147
}
141148

142-
images[platform] = image
149+
platform := "unknown/unknown"
150+
if desc.Platform != nil {
151+
platform = desc.Platform.String()
152+
}
153+
if _, exists := images[platform]; !exists {
154+
images[platform] = make([]cr.Image, 0)
155+
}
156+
157+
images[platform] = append(images[platform], image)
158+
159+
l.Debug("Downloaded a manifest",
160+
zap.String("digest", desc.Digest.String()),
161+
zap.String("platform", platform),
162+
zap.Any("annotations", desc.Annotations),
163+
)
143164
}
144165
}
145166
}
@@ -167,35 +188,35 @@ func (s *Server) downloadGithubContainer(
167188
zipper := zip.NewWriter(file)
168189
defer zipper.Close()
169190

170-
for platform, image := range images {
171-
digest, err := image.Digest()
172-
if err != nil {
173-
return "", fmt.Errorf("failed to get image digest: %s: %w",
174-
j.GetPackageUrl(), err,
175-
)
176-
}
191+
for platform, _images := range images {
192+
for _, image := range _images {
193+
digest, err := image.Digest()
194+
if err != nil {
195+
return "", fmt.Errorf("failed to get image digest: %s: %w",
196+
j.GetPackageUrl(), err,
197+
)
198+
}
177199

178-
stream, err := zipper.Create(filepath.Join(platform, digest.Hex+".tar"))
179-
if err != nil {
180-
return "", fmt.Errorf("failed to create add container tarball to file: %w", err)
181-
}
200+
stream, err := zipper.Create(filepath.Join(platform, digest.Hex+".tar"))
201+
if err != nil {
202+
return "", fmt.Errorf("failed to create add container tarball to file: %w", err)
203+
}
182204

183-
l.Debug("Downloading container image...",
184-
zap.String("digest", digest.String()),
185-
zap.String("platform", platform),
186-
)
205+
l.Debug("Archiving container manifest...",
206+
zap.String("digest", digest.String()),
207+
)
187208

188-
start := time.Now()
209+
start := time.Now()
189210

190-
if err := crtarball.Write(ref, image, stream); err != nil {
191-
return "", fmt.Errorf("failed to write container tarball: %w", err)
192-
}
211+
if err := crtarball.Write(ref, image, stream); err != nil {
212+
return "", fmt.Errorf("failed to write container tarball: %w", err)
213+
}
193214

194-
l.Info("Downloaded a container image",
195-
zap.String("digest", digest.String()),
196-
zap.String("platform", platform),
197-
zap.Duration("duration", time.Since(start)),
198-
)
215+
l.Info("Archived a container manifest",
216+
zap.String("digest", digest.String()),
217+
zap.Duration("duration", time.Since(start)),
218+
)
219+
}
199220
}
200221
}
201222

‎server/remove.go‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,27 @@ func (s *Server) RemoveDownload(ctx context.Context, path string) {
1717
switch s.cfg.SoftDelete.Downloads {
1818
default:
1919
target := filepath.Join(s.cfg.SoftDelete.Downloads, filepath.Base(path))
20-
err := os.Rename(path, target)
21-
if err == nil || errors.Is(err, os.ErrNotExist) {
22-
return
20+
if err := os.Rename(path, target); err != nil && !errors.Is(err, os.ErrNotExist) {
21+
l.Error("Failed to soft-delete downloaded file, will try hard-deleting...",
22+
zap.Error(err),
23+
zap.String("path", path),
24+
zap.String("target", target),
25+
)
2326
}
24-
l.Error("Failed to soft-delete downloaded file, will try hard-deleting...",
25-
zap.Error(err),
26-
zap.String("path", path),
27-
zap.String("target", target),
28-
)
2927
fallthrough
3028

3129
case "":
32-
err := os.Remove(path)
33-
if err == nil || errors.Is(err, os.ErrNotExist) {
34-
return
30+
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
31+
l.Error("Failed to remove downloaded file",
32+
zap.Error(err),
33+
zap.String("path", path),
34+
)
3535
}
36-
l.Error("Failed to remove downloaded file",
37-
zap.Error(err),
38-
zap.String("path", path),
39-
)
4036
}
37+
38+
l.Debug("Removed a downloaded file",
39+
zap.String("path", path),
40+
)
4141
}
4242

4343
func (s *Server) RemoveJob(ctx context.Context, j job.Job) {
@@ -46,25 +46,25 @@ func (s *Server) RemoveJob(ctx context.Context, j job.Job) {
4646
switch s.cfg.SoftDelete.Jobs {
4747
default:
4848
target := filepath.Join(s.cfg.SoftDelete.Jobs, filepath.Base(job.Path(j)))
49-
err := os.Rename(job.Path(j), target)
50-
if err == nil || errors.Is(err, os.ErrNotExist) {
51-
return
49+
if err := os.Rename(job.Path(j), target); err != nil && !errors.Is(err, os.ErrNotExist) {
50+
l.Error("Failed to soft-delete persisted job, will try hard-deleting...",
51+
zap.Error(err),
52+
zap.String("path", job.Path(j)),
53+
zap.String("target", target),
54+
)
5255
}
53-
l.Error("Failed to soft-delete persisted job, will try hard-deleting...",
54-
zap.Error(err),
55-
zap.String("path", job.Path(j)),
56-
zap.String("target", target),
57-
)
5856
fallthrough
5957

6058
case "":
61-
err := os.Remove(job.Path(j))
62-
if err == nil || errors.Is(err, os.ErrNotExist) {
63-
return
59+
if err := os.Remove(job.Path(j)); err != nil && errors.Is(err, os.ErrNotExist) {
60+
l.Error("Failed to remove persisted job",
61+
zap.Error(err),
62+
zap.String("path", job.Path(j)),
63+
)
6464
}
65-
l.Error("Failed to remove persisted job",
66-
zap.Error(err),
67-
zap.String("path", job.Path(j)),
68-
)
6965
}
66+
67+
l.Debug("Removed persisted job",
68+
zap.String("path", job.Path(j)),
69+
)
7070
}

0 commit comments

Comments
 (0)