From 2321b177fa7bf2b25bd43e923419a3a0b1fd04a8 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 09:43:48 +0000 Subject: [PATCH 1/8] Run all tests on HNS bucket with experimentation flag enabled --- cloudbuild/e2e-tests-cloudbuild.yaml | 164 +++++++++++++++++++-------- 1 file changed, 114 insertions(+), 50 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index bc1bcebc..c527dac4 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -4,7 +4,7 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} steps: - # Step 0: Generate a persistent SSH key for this build run. + # Generate a persistent SSH key for this build run. # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "generate-ssh-key" @@ -19,7 +19,7 @@ steps: cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub waitFor: ["-"] - # Step 1: Create a unique standard GCS bucket for the test run. + # Create a unique standard GCS bucket for the test run. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "create-standard-bucket" entrypoint: "gcloud" @@ -32,7 +32,7 @@ steps: - "--location=${_REGION}" waitFor: ["-"] - # Step 2: Create a unique versioned GCS bucket for the test run. + # Create a unique versioned GCS bucket for the test run. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "create-versioned-bucket" entrypoint: "gcloud" @@ -45,7 +45,7 @@ steps: - "--location=${_REGION}" waitFor: ["-"] - # Step 2a: Enable versioning on the versioned bucket. + # Enable versioning on the versioned bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "enable-bucket-versioning" entrypoint: "gcloud" @@ -58,7 +58,7 @@ steps: waitFor: - "create-versioned-bucket" - # Step 3: Create a unique HNS GCS bucket for the test run. + # Create a unique HNS GCS bucket for the test run. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "create-hns-bucket" entrypoint: "gcloud" @@ -73,7 +73,7 @@ steps: - "--uniform-bucket-level-access" waitFor: ["-"] - # Step 4: Create a GCE VM to run the tests. + # Create a GCE VM to run the tests. # The VM is created in the same zone as the buckets to test rapid storage features. # It's given the 'cloud-platform' scope to allow it to access GCS and other services. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" @@ -94,10 +94,10 @@ steps: - "--metadata=enable-oslogin=TRUE" waitFor: ["-"] - # Step 5: Run the integration tests inside the newly created VM. + # Set up the VM for integration tests. # This step uses 'gcloud compute ssh' to execute a remote script. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "run-tests-on-vm" + id: "setup-vm" entrypoint: "bash" args: - "-c" @@ -117,7 +117,7 @@ steps: # Script to be executed on the VM. # This script installs dependencies, sets environment variables, and runs pytest. - VM_SCRIPT=" + SETUP_SCRIPT=" set -e echo '--- Installing dependencies on VM ---' sudo apt-get update > /dev/null @@ -131,51 +131,95 @@ steps: # Install testing libraries explicitly, as they are not in setup.py pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null pip install -e . > /dev/null - - echo '--- Preparing test environment on VM ---' - export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' - export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' - export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' - export GCSFS_HNS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' - - export STORAGE_EMULATOR_HOST=https://storage.googleapis.com - export GCSFS_TEST_PROJECT=${PROJECT_ID} - export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} - - echo '--- Running standard tests on VM ---' - pytest -vv -s \ - --log-format='%(asctime)s %(levelname)s %(message)s' \ - --log-date-format='%H:%M:%S' \ - --color=no \ - gcsfs/ \ - --deselect gcsfs/tests/test_core.py::test_sign - - # Run Zonal and HNS tests - echo '--- Running Zonal and HNS tests on VM ---' - # Increased the ulimit temporarily to avoid too many open files error - ulimit -n 2048 - export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true" - pytest -vv -s \ - --log-format='%(asctime)s %(levelname)s %(message)s' \ - --log-date-format='%H:%M:%S' \ - --color=no \ - gcsfs/tests/test_extended_gcsfs.py \ - gcsfs/tests/test_extended_hns_gcsfs.py \ - gcsfs/tests/test_zonal_file.py " # Execute the script on the VM via SSH. - gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$VM_SCRIPT" + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT" waitFor: - "create-vm" + - "generate-ssh-key" + + # Run standard tests (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-standard-tests" + entrypoint: "gcloud" + args: + - "compute" + - "ssh" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--zone=${_ZONE}" + - "--internal-ip" + - "--ssh-key-file=/workspace/.ssh/google_compute_engine" + - |- + source env/bin/activate && \ + echo '--- Preparing test environment for standard tests ---' && \ + export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' && \ + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running standard tests on VM ---' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_core.py::test_sign + waitFor: + - "setup-vm" - "create-standard-bucket" - "enable-bucket-versioning" - - "generate-ssh-key" + + # Run zonal tests (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-zonal-tests" + entrypoint: "gcloud" + args: + - "compute" + - "ssh" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--zone=${_ZONE}" + - "--internal-ip" + - "--ssh-key-file=/workspace/.ssh/google_compute_engine" + - |- + source env/bin/activate && \ + echo '--- Preparing test environment for zonal tests ---' && \ + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running Zonal tests on VM ---' && \ + ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py + waitFor: + - "setup-vm" + + # Run HNS tests (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-hns-tests" + entrypoint: "gcloud" + args: + - "compute" + - "ssh" + - "gcsfs-test-vm-${_SHORT_BUILD_ID}" + - "--zone=${_ZONE}" + - "--internal-ip" + - "--ssh-key-file=/workspace/.ssh/google_compute_engine" + - |- + source env/bin/activate && \ + echo '--- Preparing test environment for HNS tests ---' && \ + export GCSFS_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_VERSIONED_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_ZONAL_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running HNS tests on VM ---' && \ + export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ + waitFor: + - "setup-vm" - "create-hns-bucket" # --- Cleanup Steps --- - # Step 6: Clean up the SSH key from the OS Login profile. + # Clean up the SSH key from the OS Login profile. # This step is crucial to prevent key accumulation. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "cleanup-ssh-key" @@ -187,9 +231,11 @@ steps: gcloud compute os-login ssh-keys remove \ --key-file=/workspace/gcb_ssh_key.pub || true waitFor: - - "run-tests-on-vm" + - "run-standard-tests" + - "run-zonal-tests" + - "run-hns-tests" - # Step 7: Delete the GCE VM. + # Delete the GCE VM. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-vm" entrypoint: "gcloud" @@ -203,7 +249,7 @@ steps: waitFor: - "cleanup-ssh-key" - # Step 8: Delete the standard GCS bucket. + # Delete the standard GCS bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-standard-bucket" entrypoint: "gcloud" @@ -215,9 +261,9 @@ steps: "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}", ] waitFor: - - "run-tests-on-vm" + - "run-standard-tests" - # Step 9: Delete the versioned GCS bucket. + # Delete the versioned GCS bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-versioned-bucket" entrypoint: "gcloud" @@ -229,7 +275,16 @@ steps: "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}", ] waitFor: - - "run-tests-on-vm" + - "run-standard-tests" + + # Delete the HNS GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-hns-bucket" + entrypoint: "gcloud" + args: + ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] + waitFor: + - "run-hns-tests" # Step 10: Delete the HNS GCS bucket. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" @@ -238,7 +293,16 @@ steps: args: ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] waitFor: - - "run-tests-on-vm" + - "run-standard-tests" + + # Delete the HNS GCS bucket. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "delete-hns-bucket" + entrypoint: "gcloud" + args: + ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] + waitFor: + - "run-hns-tests" timeout: "3600s" # 60 minutes From 271745dd3e395e8171c5ff9302a1488412f14296 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 10:09:02 +0000 Subject: [PATCH 2/8] refactor cloudbuild yaml file --- cloudbuild/e2e-tests-cloudbuild.yaml | 213 ++++++++++----------------- 1 file changed, 77 insertions(+), 136 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index c527dac4..aad4e1db 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -19,58 +19,25 @@ steps: cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub waitFor: ["-"] - # Create a unique standard GCS bucket for the test run. + # Create all necessary GCS buckets in parallel. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-standard-bucket" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - waitFor: ["-"] - - # Create a unique versioned GCS bucket for the test run. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-versioned-bucket" - entrypoint: "gcloud" + id: "create-buckets" + entrypoint: "bash" args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - waitFor: ["-"] + - "-c" + - | + set -e + echo "--- Creating standard bucket ---" + gcloud storage buckets create gs://gcsfs-test-standard-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} & + echo "--- Creating versioned bucket ---" + gcloud storage buckets create gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} & + echo "--- Creating HNS bucket ---" + gcloud storage buckets create gs://gcsfs-test-hns-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} --enable-hierarchical-namespace --uniform-bucket-level-access & - # Enable versioning on the versioned bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "enable-bucket-versioning" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "update" - - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" - - "--versioning" - waitFor: - - "create-versioned-bucket" + wait - # Create a unique HNS GCS bucket for the test run. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-hns-bucket" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - - "--enable-hierarchical-namespace" - - "--uniform-bucket-level-access" + echo "--- Enabling versioning on versioned bucket ---" + gcloud storage buckets update gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} --versioning waitFor: ["-"] # Create a GCE VM to run the tests. @@ -142,80 +109,73 @@ steps: # Run standard tests (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-standard-tests" - entrypoint: "gcloud" + entrypoint: "bash" args: - - "compute" - - "ssh" - - "gcsfs-test-vm-${_SHORT_BUILD_ID}" - - "--zone=${_ZONE}" - - "--internal-ip" - - "--ssh-key-file=/workspace/.ssh/google_compute_engine" - - |- - source env/bin/activate && \ - echo '--- Preparing test environment for standard tests ---' && \ - export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' && \ - export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' && \ - export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ - export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ - export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ - export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ - echo '--- Running standard tests on VM ---' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_core.py::test_sign + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for standard tests ---' && \ + export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' && \ + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running standard tests on VM ---' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_core.py::test_sign + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: - "setup-vm" - - "create-standard-bucket" - - "enable-bucket-versioning" + - "create-buckets" # Run zonal tests (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-zonal-tests" - entrypoint: "gcloud" + entrypoint: "bash" args: - - "compute" - - "ssh" - - "gcsfs-test-vm-${_SHORT_BUILD_ID}" - - "--zone=${_ZONE}" - - "--internal-ip" - - "--ssh-key-file=/workspace/.ssh/google_compute_engine" - - |- - source env/bin/activate && \ - echo '--- Preparing test environment for zonal tests ---' && \ - export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ - export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ - export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ - export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ - echo '--- Running Zonal tests on VM ---' && \ - ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for zonal tests ---' && \ + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running Zonal tests on VM ---' && \ + ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: - "setup-vm" # Run HNS tests (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-hns-tests" - entrypoint: "gcloud" + entrypoint: "bash" args: - - "compute" - - "ssh" - - "gcsfs-test-vm-${_SHORT_BUILD_ID}" - - "--zone=${_ZONE}" - - "--internal-ip" - - "--ssh-key-file=/workspace/.ssh/google_compute_engine" - - |- - source env/bin/activate && \ - echo '--- Preparing test environment for HNS tests ---' && \ - export GCSFS_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ - export GCSFS_TEST_VERSIONED_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ - export GCSFS_ZONAL_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ - export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ - export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ - export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ - echo '--- Running HNS tests on VM ---' && \ - export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for HNS tests ---' && \ + export GCSFS_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_VERSIONED_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_ZONAL_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running HNS tests on VM ---' && \ + export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: - "setup-vm" - - "create-hns-bucket" + - "create-buckets" # --- Cleanup Steps --- @@ -249,41 +209,22 @@ steps: waitFor: - "cleanup-ssh-key" - # Delete the standard GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-standard-bucket" - entrypoint: "gcloud" - args: - [ - "storage", - "rm", - "--recursive", - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}", - ] - waitFor: - - "run-standard-tests" - - # Delete the versioned GCS bucket. + # Delete all GCS buckets in parallel. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-versioned-bucket" - entrypoint: "gcloud" + id: "delete-buckets" + entrypoint: "bash" args: - [ - "storage", - "rm", - "--recursive", - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}", - ] + - "-c" + - | + set -e + echo "--- Deleting test buckets in parallel ---" + gcloud storage rm --recursive gs://gcsfs-test-standard-${_SHORT_BUILD_ID} & + gcloud storage rm --recursive gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} & + gcloud storage rm --recursive gs://gcsfs-test-hns-${_SHORT_BUILD_ID} & + wait waitFor: - "run-standard-tests" - - # Delete the HNS GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-hns-bucket" - entrypoint: "gcloud" - args: - ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] - waitFor: + - "run-zonal-tests" - "run-hns-tests" # Step 10: Delete the HNS GCS bucket. From cb9fef5681efa51403d645de111c6d54bd762768 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 13:47:35 +0000 Subject: [PATCH 3/8] Exclude zonal and version tests on HNS bucket --- cloudbuild/e2e-tests-cloudbuild.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index aad4e1db..55cdf097 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -118,7 +118,6 @@ steps: echo '--- Preparing test environment for standard tests ---' && \ export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' && \ export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' && \ - export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ @@ -162,15 +161,13 @@ steps: TEST_SCRIPT=" source env/bin/activate && \ echo '--- Preparing test environment for HNS tests ---' && \ - export GCSFS_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ - export GCSFS_TEST_VERSIONED_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ - export GCSFS_ZONAL_TEST_BUCKET='gs://gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running HNS tests on VM ---' && \ export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py " gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: From 3a5bdad82c34bbacc7dc52cd008e9363e2c972e7 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 15:07:15 +0000 Subject: [PATCH 4/8] exclude test_sign and resolve conflicts --- cloudbuild/e2e-tests-cloudbuild.yaml | 29 ++++++---------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 55cdf097..79f19703 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -41,7 +41,7 @@ steps: waitFor: ["-"] # Create a GCE VM to run the tests. - # The VM is created in the same zone as the buckets to test rapid storage features. + # The VM is created in the same zone as the zonal bucket to test rapid storage features. # It's given the 'cloud-platform' scope to allow it to access GCS and other services. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "create-vm" @@ -62,7 +62,7 @@ steps: waitFor: ["-"] # Set up the VM for integration tests. - # This step uses 'gcloud compute ssh' to execute a remote script. + # This step uses 'gcloud compute ssh' to execute a script. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "setup-vm" entrypoint: "bash" @@ -129,7 +129,7 @@ steps: - "setup-vm" - "create-buckets" - # Run zonal tests (in parallel with other tests). + # Run extended tests on Zonal bucket (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-zonal-tests" entrypoint: "bash" @@ -151,7 +151,7 @@ steps: waitFor: - "setup-vm" - # Run HNS tests (in parallel with other tests). + # Run tests on HNS bucket (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "run-hns-tests" entrypoint: "bash" @@ -162,12 +162,13 @@ steps: source env/bin/activate && \ echo '--- Preparing test environment for HNS tests ---' && \ export GCSFS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_HNS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running HNS tests on VM ---' && \ export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py --deselect gcsfs/tests/test_core.py::test_sign " gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: @@ -224,24 +225,6 @@ steps: - "run-zonal-tests" - "run-hns-tests" - # Step 10: Delete the HNS GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-hns-bucket" - entrypoint: "gcloud" - args: - ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] - waitFor: - - "run-standard-tests" - - # Delete the HNS GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-hns-bucket" - entrypoint: "gcloud" - args: - ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] - waitFor: - - "run-hns-tests" - timeout: "3600s" # 60 minutes options: From 626a913fe98bbb04866d2c9617a6675c0c9b753d Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 15:22:14 +0000 Subject: [PATCH 5/8] update waitFor for zonal tests --- cloudbuild/e2e-tests-cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 79f19703..7ef74e10 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -150,6 +150,7 @@ steps: gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: - "setup-vm" + - "create-buckets" # Run tests on HNS bucket (in parallel with other tests). - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" From 1cffc85b04ebfc3c116767b88d62102ded046b46 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Wed, 24 Dec 2025 16:09:10 +0000 Subject: [PATCH 6/8] fix failing test in test_core.py rm doesn't delete the empty folders yet on HNS buckets, so if same destination folder name is used for multiple tests they would fail with conflict as destination already exists. This error is thrown from the rename_folder HNS API --- gcsfs/tests/test_core.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index b0f6b8bd..740737c4 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -532,12 +532,9 @@ def test_move(gcs): assert not gcs.exists(fn) -@pytest.mark.parametrize("slash_from", ([False, True])) -def test_move_recursive(gcs, slash_from): +def test_move_recursive_no_slash(gcs): # See issue #489 dir_from = TEST_BUCKET + "/nested" - if slash_from: - dir_from += "/" dir_to = TEST_BUCKET + "/new_name" gcs.mv(dir_from, dir_to, recursive=True) @@ -545,6 +542,16 @@ def test_move_recursive(gcs, slash_from): assert gcs.ls(dir_to) == [dir_to + "/file1", dir_to + "/file2", dir_to + "/nested2"] +def test_move_recursive_with_slash(gcs): + # See issue #489 + dir_from = TEST_BUCKET + "/nested/" + dir_to = TEST_BUCKET + "/new_name_with_slash" + + gcs.mv(dir_from, dir_to, recursive=True) + assert not gcs.exists(dir_from.rstrip("/")) + assert gcs.ls(dir_to) == [dir_to + "/file1", dir_to + "/file2", dir_to + "/nested2"] + + def test_cat_file(gcs): fn = TEST_BUCKET + "/test/accounts.1.json" data = gcs.cat_file(fn) From c4ae7ec70b97f2a3270946ac9570277f00de5842 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Fri, 26 Dec 2025 09:38:26 +0000 Subject: [PATCH 7/8] resolve merge conflict --- cloudbuild/e2e-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 7ef74e10..9df6a102 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -145,7 +145,7 @@ steps: export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running Zonal tests on VM ---' && \ ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py gcsfs/tests/test_zonal_file.py " gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: From 7a7a4bb926e5b51de3075f5428da6f24294d335f Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Fri, 26 Dec 2025 10:01:12 +0000 Subject: [PATCH 8/8] exclude zonal_file tests on HNS bucket --- cloudbuild/e2e-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 9df6a102..09d17638 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -169,7 +169,7 @@ steps: export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running HNS tests on VM ---' && \ export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ - pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py --deselect gcsfs/tests/test_core.py::test_sign + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py --deselect gcsfs/tests/test_core.py::test_sign --deselect gcsfs/tests/test_zonal_file.py " gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" waitFor: