From 0ce754a78f3eecae36c1c46518c31eb568a22a50 Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Tue, 21 Oct 2025 19:40:29 +0200 Subject: [PATCH 1/2] kubetest2-kops: drop gsutil for GCS bucket lifecycle Use GCS Go SDK to operate the state and discovery buckets Signed-off-by: Arnaud Meukam --- tests/e2e/kubetest2-kops/gce/gcs.go | 92 ++++++++++++++++----------- tests/e2e/kubetest2-kops/gce/zones.go | 6 ++ 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/tests/e2e/kubetest2-kops/gce/gcs.go b/tests/e2e/kubetest2-kops/gce/gcs.go index 067290d72057c..2a2c7add5d2f9 100644 --- a/tests/e2e/kubetest2-kops/gce/gcs.go +++ b/tests/e2e/kubetest2-kops/gce/gcs.go @@ -17,16 +17,24 @@ limitations under the License. package gce import ( + "context" "crypto/rand" "encoding/hex" + "errors" + "fmt" "os" "strings" - "time" + "cloud.google.com/go/iam" + "cloud.google.com/go/storage" "k8s.io/klog/v2" "sigs.k8s.io/kubetest2/pkg/exec" ) +const ( + defaultRegion = "us-central1" +) + func GCSBucketName(projectID, prefix string) string { var s string if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 { @@ -41,57 +49,69 @@ func GCSBucketName(projectID, prefix string) string { } func EnsureGCSBucket(bucketPath, projectID string, public bool) error { - lsArgs := []string{ - "gsutil", "ls", "-b", - } - if projectID != "" { - lsArgs = append(lsArgs, "-p", projectID) + // TODO: Detect the GCP region used + return EnsureGCSBucketWithRegion(bucketPath, projectID, defaultRegion, public) +} + +func EnsureGCSBucketWithRegion(bucketPath, projectID string, region string, public bool) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return fmt.Errorf("failed to create storage client: %w", err) } - lsArgs = append(lsArgs, bucketPath) + defer client.Close() + + // Extract bucket name from gs:// path if the bucket's URI is provided + bucketName := strings.TrimPrefix(bucketPath, "gs://") + bucketName = strings.TrimSuffix(bucketName, "/") - klog.Info(strings.Join(lsArgs, " ")) - cmd := exec.Command(lsArgs[0], lsArgs[1:]...) + bucket := client.Bucket(bucketName) - output, err := exec.CombinedOutputLines(cmd) + // Check if bucket exists + klog.Infof("Checking if bucket %s exists", bucketName) + _, err = bucket.Attrs(ctx) if err == nil { + klog.Infof("Bucket %s already exists", bucketName) return nil - } else if len(output) != 1 || !strings.Contains(output[0], "BucketNotFound") { - klog.Info(output) - return err } - mbArgs := []string{ - "gsutil", "mb", - } - if projectID != "" { - mbArgs = append(mbArgs, "-p", projectID) + // If error is not "bucket doesn't exist", return error + if errors.Is(err, storage.ErrBucketNotExist) { + return fmt.Errorf("error checking bucket: %w", err) } - mbArgs = append(mbArgs, bucketPath) - klog.Info(strings.Join(mbArgs, " ")) - cmd = exec.Command(mbArgs[0], mbArgs[1:]...) + // Create the bucket + klog.Infof("Creating bucket %s in project %s", bucketName, projectID) + bucketAttrs := &storage.BucketAttrs{ + Location: region, + LocationType: "region", + UniformBucketLevelAccess: storage.UniformBucketLevelAccess{ + Enabled: true, + }, + SoftDeletePolicy: &storage.SoftDeletePolicy{ + RetentionDuration: 0, + }, + } - exec.InheritOutput(cmd) - err = cmd.Run() - if err != nil { - return err + if err := bucket.Create(ctx, projectID, bucketAttrs); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) } if public { - iamArgs := []string{ - "gsutil", "iam", "ch", "allUsers:objectViewer", - } - iamArgs = append(iamArgs, bucketPath) - klog.Info(strings.Join(iamArgs, " ")) - // GCS APIs are strongly consistent but this should help with flakes - time.Sleep(10 * time.Second) - cmd = exec.Command(iamArgs[0], iamArgs[1:]...) - exec.InheritOutput(cmd) - err = cmd.Run() + klog.Infof("Making bucket %s public", bucketName) + policy, err := bucket.IAM().Policy(ctx) if err != nil { - return err + return fmt.Errorf("failed to get bucket IAM policy: %w", err) + } + + // Add allUsers as objectViewer + policy.Add(iam.AllUsers, "roles/storage.objectViewer") + + if err := bucket.IAM().SetPolicy(ctx, policy); err != nil { + return fmt.Errorf("failed to set bucket IAM policy: %w", err) } } + return nil } diff --git a/tests/e2e/kubetest2-kops/gce/zones.go b/tests/e2e/kubetest2-kops/gce/zones.go index 04198db2ac944..8dcb069eafccc 100644 --- a/tests/e2e/kubetest2-kops/gce/zones.go +++ b/tests/e2e/kubetest2-kops/gce/zones.go @@ -34,6 +34,12 @@ var allZones = []string{ "us-east4-a", "us-east4-b", "us-east4-c", + "us-east5-a", + "us-east5-b", + "us-east5-c", + "us-south1-a", + "us-south1-b", + "us-south1-c", "us-west1-a", "us-west1-b", "us-west1-c", From 8e38ca7dc69d1f4455579508bf5703cb3a1d4305 Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Tue, 21 Oct 2025 19:42:44 +0200 Subject: [PATCH 2/2] Bump golang packages Signed-off-by: Arnaud Meukam --- tests/e2e/go.mod | 10 +++++----- tests/e2e/go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 8e69b6c96c258..83d932a50efab 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -5,6 +5,8 @@ go 1.25.0 replace k8s.io/kops => ../../. require ( + cloud.google.com/go/iam v1.5.2 + cloud.google.com/go/storage v1.57.0 github.com/aws/aws-sdk-go-v2 v1.39.2 github.com/aws/aws-sdk-go-v2/config v1.31.12 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 @@ -27,13 +29,11 @@ require ( require ( cel.dev/expr v0.24.0 // indirect - cloud.google.com/go v0.120.0 // indirect + cloud.google.com/go v0.121.6 // indirect cloud.google.com/go/auth v0.16.5 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect - cloud.google.com/go/storage v1.50.0 // indirect cuelabs.dev/go/oci/ociregistry v0.0.0-20240404174027-a39bec0462d2 // indirect cuelang.org/go v0.9.2 // indirect dario.cat/mergo v1.0.1 // indirect @@ -55,8 +55,8 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 // indirect github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/MakeNowJust/heredoc/v2 v2.0.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.5 // indirect diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 278a6f5821eaa..b71f342169f93 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -9,8 +9,8 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= -cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= @@ -34,8 +34,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= -cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/storage v1.57.0 h1:4g7NB7Ta7KetVbOMpCqy89C+Vg5VE8scqlSHUPm7Rds= +cloud.google.com/go/storage v1.57.0/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg= cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= cuelabs.dev/go/oci/ociregistry v0.0.0-20240404174027-a39bec0462d2 h1:BnG6pr9TTr6CYlrJznYUDj6V7xldD1W+1iXPum0wT/w= @@ -102,12 +102,12 @@ github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0 h1:lwL1vLWmdBJ5h+StMEN github.com/GoogleCloudPlatform/k8s-cloud-provider v1.25.0/go.mod h1:UTfhBnADaj2rybPT049NScSh7Eall3u2ib43wmz3deg= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0/go.mod h1:ZV4VOm0/eHR06JLrXWe09068dHpr3TRpY9Uo7T+anuA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0 h1:nNMpRpnkWDAaqcpxMJvxa/Ud98gjbYwayJY4/9bdjiU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 h1:ig/FpDD2JofP/NExKQUbn7uOSZzJAQqogfqluZK4ed4= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A= github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -884,8 +884,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 h1:m639+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0/go.mod h1:LjReUci/F4BUyv+y4dwnq3h/26iNOeC3wAIqgvTIZVo= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=