From 5744b37c79496a57a69fda5afb7ef01f0cbb9539 Mon Sep 17 00:00:00 2001 From: kites262 Date: Mon, 3 Nov 2025 21:48:11 +0800 Subject: [PATCH] fix: list images api - time parse err - d.Walk now contains folders --- registry/storage/driver/cos/cos.go | 75 ++++++++++++-------- registry/storage/driver/cos/parser.go | 1 + registry/storage/driver/cos/utils.go | 86 +++++++++++++++++++++++ registry/storage/driver/cos/utils_test.go | 51 ++++++++++++++ 4 files changed, 185 insertions(+), 28 deletions(-) create mode 100644 registry/storage/driver/cos/utils.go create mode 100644 registry/storage/driver/cos/utils_test.go diff --git a/registry/storage/driver/cos/cos.go b/registry/storage/driver/cos/cos.go index edc89c39fd0..6c9e1c84f20 100644 --- a/registry/storage/driver/cos/cos.go +++ b/registry/storage/driver/cos/cos.go @@ -5,11 +5,6 @@ import ( "context" "errors" "fmt" - "github.com/distribution/distribution/v3/internal/dcontext" - storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" - "github.com/distribution/distribution/v3/registry/storage/driver/base" - "github.com/distribution/distribution/v3/registry/storage/driver/factory" - "github.com/tencentyun/cos-go-sdk-v5" "io" "net/http" "net/url" @@ -18,6 +13,12 @@ import ( "strings" "sync" "time" + + "github.com/distribution/distribution/v3/internal/dcontext" + storagedriver "github.com/distribution/distribution/v3/registry/storage/driver" + "github.com/distribution/distribution/v3/registry/storage/driver/base" + "github.com/distribution/distribution/v3/registry/storage/driver/factory" + "github.com/tencentyun/cos-go-sdk-v5" ) const ( @@ -286,7 +287,10 @@ func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, } // file info size, err := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64) - modTime, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified")) + if err != nil { + return nil, err + } + modTime, err := ParseTime(res.Header.Get("Last-Modified")) if err != nil { return nil, err } @@ -374,7 +378,7 @@ func (d *driver) copy(ctx context.Context, sourcePath, destPath string) error { return nil } // file size is larger than 32MB - v, _, err := d.cosClient.Object.InitiateMultipartUpload(ctx, key, &cos.InitiateMultipartUploadOptions{ + v, _, _ := d.cosClient.Object.InitiateMultipartUpload(ctx, key, &cos.InitiateMultipartUploadOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ ContentType: contentType, }, @@ -524,6 +528,8 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta var retError error // the most recent skip directory to avoid walking over undesirable files var prevSkipDir string + // the most recent directory walked for de-duplication + var prevDir string key := d.pathToKey(from) opt := &cos.BucketGetOptions{ @@ -532,7 +538,7 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta Marker: d.pathToKey(startAfter), } ctx, done := dcontext.WithTrace(parentCtx) - defer done("cos.Bucket.Get(%s)", opt) + defer done("cos.Bucket.Get(%+v)", opt) isTruncated := true for isTruncated { // list all the objects @@ -541,30 +547,45 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta return err } walkInfos := make([]storagedriver.FileInfoInternal, 0, len(res.Contents)) + prevDir = from for _, content := range res.Contents { - if strings.HasSuffix(content.Key, "/") { // directory + // COS returns only objects, without directories + path := d.keyToPath(content.Key) + + // get a list of all directories between the previous + dirs := DirectoryDiff(prevDir, path) + for _, dir := range dirs { walkInfos = append(walkInfos, storagedriver.FileInfoInternal{ FileInfoFields: storagedriver.FileInfoFields{ IsDir: true, - Path: strings.TrimRight(content.Key, "/"), - }, - }) - } else { // file object - // last modification time - modTime, err := time.Parse(time.RFC1123, content.LastModified) - if err != nil { - return err - } - walkInfos = append(walkInfos, storagedriver.FileInfoInternal{ - FileInfoFields: storagedriver.FileInfoFields{ - IsDir: false, - Size: content.Size, - ModTime: modTime, - Path: content.Key, + Path: dir, }, }) + prevDir = dir + } + + // corner case + if strings.HasSuffix(path, "/") { + continue } + + modTime, err := ParseTime(content.LastModified) + if err != nil { + return err + } + + walkInfos = append(walkInfos, storagedriver.FileInfoInternal{ + FileInfoFields: storagedriver.FileInfoFields{ + IsDir: false, + Size: content.Size, + ModTime: modTime, + Path: path, + }, + }) } + + // according to the files, append directories + isTruncated = res.IsTruncated opt.Marker = res.NextMarker // iterative @@ -600,14 +621,12 @@ func (d *driver) doWalk(parentCtx context.Context, objectCount *int64, from, sta // add the prefix func (d *driver) pathToKey(path string) string { - // Important! delete the root prefix - newPath := strings.TrimPrefix(path, d.rootDirectory) - return strings.TrimLeft(strings.TrimRight(d.rootDirectory, "/")+newPath, "/") + return PathToKey(d.rootDirectory, path) } // remove the prefix func (d *driver) keyToPath(key string) string { - return "/" + strings.TrimRight(strings.TrimPrefix(key, d.rootDirectory), "/") + return KeyToPath(d.rootDirectory, key) } var _ storagedriver.FileWriter = &writer{} diff --git a/registry/storage/driver/cos/parser.go b/registry/storage/driver/cos/parser.go index 6393432d19e..607955989d5 100644 --- a/registry/storage/driver/cos/parser.go +++ b/registry/storage/driver/cos/parser.go @@ -2,6 +2,7 @@ package cos import ( "errors" + "github.com/mitchellh/mapstructure" ) diff --git a/registry/storage/driver/cos/utils.go b/registry/storage/driver/cos/utils.go new file mode 100644 index 00000000000..d29a9439d62 --- /dev/null +++ b/registry/storage/driver/cos/utils.go @@ -0,0 +1,86 @@ +package cos + +import ( + "path/filepath" + "slices" + "strings" + "time" +) + +func ParseTime(timeStr string) (time.Time, error) { + var timeObj time.Time + timeObj, err := time.Parse(time.RFC3339, timeStr) + if err != nil { + timeObj, err = time.Parse(time.RFC1123, timeStr) + if err != nil { + return time.Time{}, err + } + } + return timeObj, nil +} + +// DirectoryDiff finds all directories that are not in common between +// the previous and current paths in sorted order. +// +// # Examples +// +// DirectoryDiff("/path/to/folder", "/path/to/folder/folder/file") +// // => [ "/path/to/folder/folder" ] +// +// DirectoryDiff("/path/to/folder/folder1", "/path/to/folder/folder2/file") +// // => [ "/path/to/folder/folder2" ] +// +// DirectoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/file") +// // => [ "/path/to/folder/folder2" ] +// +// DirectoryDiff("/path/to/folder/folder1/file", "/path/to/folder/folder2/folder1/file") +// // => [ "/path/to/folder/folder2", "/path/to/folder/folder2/folder1" ] +// +// DirectoryDiff("/", "/path/to/folder/folder/file") +// // => [ "/path", "/path/to", "/path/to/folder", "/path/to/folder/folder" ] +func DirectoryDiff(prev, current string) []string { + var paths []string + + if prev == "" || current == "" { + return paths + } + + parent := current + for { + parent = filepath.Dir(parent) + if parent == "/" || parent == prev || strings.HasPrefix(prev+"/", parent+"/") { + break + } + paths = append(paths, parent) + } + slices.Reverse(paths) + return paths +} + +// PathToKey 将路径转换为存储键 +func PathToKey(rootDir, path string) string { + rootDir = strings.Trim(rootDir, "/") + path = strings.Trim(path, "/") + // Important! delete the root prefix if existed + path = strings.TrimLeft(strings.TrimPrefix(path, rootDir), "/") + if rootDir == "" { + return path + } + if path == "" { + return rootDir + } + return rootDir + "/" + path +} + +// KeyToPath 将存储键转换为路径 +func KeyToPath(rootDir, key string) string { + rootDir = strings.Trim(rootDir, "/") + key = strings.Trim(key, "/") + if rootDir == "" { + return "/" + key + } + if key == "" { + return "" + } + return "/" + strings.TrimLeft(strings.TrimPrefix(key, rootDir), "/") +} diff --git a/registry/storage/driver/cos/utils_test.go b/registry/storage/driver/cos/utils_test.go new file mode 100644 index 00000000000..ab810f11dac --- /dev/null +++ b/registry/storage/driver/cos/utils_test.go @@ -0,0 +1,51 @@ +package cos + +import "testing" + +func TestPathToKey(t *testing.T) { + cases := []struct { + root string + path string + key string + }{ + {root: "", path: "/a/b/c.txt", key: "a/b/c.txt"}, + {root: "", path: "a/b/c.txt", key: "a/b/c.txt"}, + {root: "", path: "/", key: ""}, + {root: "rootdir", path: "/a/b/c.txt", key: "rootdir/a/b/c.txt"}, + {root: "rootdir", path: "a/b/c.txt", key: "rootdir/a/b/c.txt"}, + {root: "rootdir", path: "/", key: "rootdir"}, + {root: "dup-rootdir", path: "/dup-rootdir/a/b/c.txt", key: "dup-rootdir/a/b/c.txt"}, + {root: "dup-rootdir", path: "dup-rootdir/a/b/c.txt", key: "dup-rootdir/a/b/c.txt"}, + } + + for _, c := range cases { + key := PathToKey(c.root, c.path) + if key != c.key { + t.Errorf("PathToKey(%q, %q) = %q; want %q", c.root, c.path, key, c.key) + } + } +} + +func TestKeyToPath(t *testing.T) { + cases := []struct { + root string + key string + path string + }{ + {root: "", key: "a/b/c.txt", path: "/a/b/c.txt"}, + {root: "", key: "a/b/c.txt", path: "/a/b/c.txt"}, + {root: "", key: "/", path: "/"}, + {root: "rootdir", key: "rootdir/a/b/c.txt", path: "/a/b/c.txt"}, + {root: "rootdir", key: "rootdir/a/b/c.txt", path: "/a/b/c.txt"}, + {root: "rootdir", key: "rootdir", path: "/"}, + {root: "dup-rootdir", key: "dup-rootdir/a/b/c.txt", path: "/a/b/c.txt"}, + {root: "dup-rootdir", key: "dup-rootdir/a/b/c.txt", path: "/a/b/c.txt"}, + } + + for _, c := range cases { + path := KeyToPath(c.root, c.key) + if path != c.path { + t.Errorf("KeyToPath(%q, %q) = %q; want %q", c.root, c.key, path, c.path) + } + } +}