diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index bc1bcebc..09d17638 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -4,7 +4,7 @@ substitutions: _SHORT_BUILD_ID: ${BUILD_ID:0:8} steps: - # Step 0: Generate a persistent SSH key for this build run. + # Generate a persistent SSH key for this build run. # This prevents gcloud from adding a new key to the OS Login profile on every ssh/scp command. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "generate-ssh-key" @@ -19,62 +19,29 @@ steps: cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub waitFor: ["-"] - # Step 1: Create a unique standard GCS bucket for the test run. + # Create all necessary GCS buckets in parallel. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-standard-bucket" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - waitFor: ["-"] - - # Step 2: Create a unique versioned GCS bucket for the test run. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-versioned-bucket" - entrypoint: "gcloud" + id: "create-buckets" + entrypoint: "bash" args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - waitFor: ["-"] + - "-c" + - | + set -e + echo "--- Creating standard bucket ---" + gcloud storage buckets create gs://gcsfs-test-standard-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} & + echo "--- Creating versioned bucket ---" + gcloud storage buckets create gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} & + echo "--- Creating HNS bucket ---" + gcloud storage buckets create gs://gcsfs-test-hns-${_SHORT_BUILD_ID} --project=${PROJECT_ID} --location=${_REGION} --enable-hierarchical-namespace --uniform-bucket-level-access & - # Step 2a: Enable versioning on the versioned bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "enable-bucket-versioning" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "update" - - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}" - - "--versioning" - waitFor: - - "create-versioned-bucket" + wait - # Step 3: Create a unique HNS GCS bucket for the test run. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-hns-bucket" - entrypoint: "gcloud" - args: - - "storage" - - "buckets" - - "create" - - "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}" - - "--project=${PROJECT_ID}" - - "--location=${_REGION}" - - "--enable-hierarchical-namespace" - - "--uniform-bucket-level-access" + echo "--- Enabling versioning on versioned bucket ---" + gcloud storage buckets update gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} --versioning waitFor: ["-"] - # Step 4: Create a GCE VM to run the tests. - # The VM is created in the same zone as the buckets to test rapid storage features. + # Create a GCE VM to run the tests. + # The VM is created in the same zone as the zonal bucket to test rapid storage features. # It's given the 'cloud-platform' scope to allow it to access GCS and other services. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "create-vm" @@ -94,10 +61,10 @@ steps: - "--metadata=enable-oslogin=TRUE" waitFor: ["-"] - # Step 5: Run the integration tests inside the newly created VM. - # This step uses 'gcloud compute ssh' to execute a remote script. + # Set up the VM for integration tests. + # This step uses 'gcloud compute ssh' to execute a script. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "run-tests-on-vm" + id: "setup-vm" entrypoint: "bash" args: - "-c" @@ -117,7 +84,7 @@ steps: # Script to be executed on the VM. # This script installs dependencies, sets environment variables, and runs pytest. - VM_SCRIPT=" + SETUP_SCRIPT=" set -e echo '--- Installing dependencies on VM ---' sudo apt-get update > /dev/null @@ -131,51 +98,87 @@ steps: # Install testing libraries explicitly, as they are not in setup.py pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null pip install -e . > /dev/null - - echo '--- Preparing test environment on VM ---' - export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' - export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' - export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' - export GCSFS_HNS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' - - export STORAGE_EMULATOR_HOST=https://storage.googleapis.com - export GCSFS_TEST_PROJECT=${PROJECT_ID} - export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} - - echo '--- Running standard tests on VM ---' - pytest -vv -s \ - --log-format='%(asctime)s %(levelname)s %(message)s' \ - --log-date-format='%H:%M:%S' \ - --color=no \ - gcsfs/ \ - --deselect gcsfs/tests/test_core.py::test_sign - - # Run Zonal and HNS tests - echo '--- Running Zonal and HNS tests on VM ---' - # Increased the ulimit temporarily to avoid too many open files error - ulimit -n 2048 - export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT="true" - pytest -vv -s \ - --log-format='%(asctime)s %(levelname)s %(message)s' \ - --log-date-format='%H:%M:%S' \ - --color=no \ - gcsfs/tests/test_extended_gcsfs.py \ - gcsfs/tests/test_extended_hns_gcsfs.py \ - gcsfs/tests/test_zonal_file.py " # Execute the script on the VM via SSH. - gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$VM_SCRIPT" + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT" waitFor: - "create-vm" - - "create-standard-bucket" - - "enable-bucket-versioning" - "generate-ssh-key" - - "create-hns-bucket" + + # Run standard tests (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-standard-tests" + entrypoint: "bash" + args: + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for standard tests ---' && \ + export GCSFS_TEST_BUCKET='gcsfs-test-standard-${_SHORT_BUILD_ID}' && \ + export GCSFS_TEST_VERSIONED_BUCKET='gcsfs-test-versioned-${_SHORT_BUILD_ID}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running standard tests on VM ---' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_core.py::test_sign + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" + waitFor: + - "setup-vm" + - "create-buckets" + + # Run extended tests on Zonal bucket (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-zonal-tests" + entrypoint: "bash" + args: + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for zonal tests ---' && \ + export GCSFS_ZONAL_TEST_BUCKET='${_GCSFS_ZONAL_TEST_BUCKET}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running Zonal tests on VM ---' && \ + ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py gcsfs/tests/test_zonal_file.py + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" + waitFor: + - "setup-vm" + - "create-buckets" + + # Run tests on HNS bucket (in parallel with other tests). + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-hns-tests" + entrypoint: "bash" + args: + - "-c" + - | + TEST_SCRIPT=" + source env/bin/activate && \ + echo '--- Preparing test environment for HNS tests ---' && \ + export GCSFS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export GCSFS_HNS_TEST_BUCKET='gcsfs-test-hns-${_SHORT_BUILD_ID}' && \ + export STORAGE_EMULATOR_HOST=https://storage.googleapis.com && \ + export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ + export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ + echo '--- Running HNS tests on VM ---' && \ + export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/ --deselect gcsfs/tests/test_extended_gcsfs.py --deselect gcsfs/tests/test_core_versioned.py --deselect gcsfs/tests/test_core.py::test_sign --deselect gcsfs/tests/test_zonal_file.py + " + gcloud compute ssh gcsfs-test-vm-${_SHORT_BUILD_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$TEST_SCRIPT" + waitFor: + - "setup-vm" + - "create-buckets" # --- Cleanup Steps --- - # Step 6: Clean up the SSH key from the OS Login profile. + # Clean up the SSH key from the OS Login profile. # This step is crucial to prevent key accumulation. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "cleanup-ssh-key" @@ -187,9 +190,11 @@ steps: gcloud compute os-login ssh-keys remove \ --key-file=/workspace/gcb_ssh_key.pub || true waitFor: - - "run-tests-on-vm" + - "run-standard-tests" + - "run-zonal-tests" + - "run-hns-tests" - # Step 7: Delete the GCE VM. + # Delete the GCE VM. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "delete-vm" entrypoint: "gcloud" @@ -203,42 +208,23 @@ steps: waitFor: - "cleanup-ssh-key" - # Step 8: Delete the standard GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-standard-bucket" - entrypoint: "gcloud" - args: - [ - "storage", - "rm", - "--recursive", - "gs://gcsfs-test-standard-${_SHORT_BUILD_ID}", - ] - waitFor: - - "run-tests-on-vm" - - # Step 9: Delete the versioned GCS bucket. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-versioned-bucket" - entrypoint: "gcloud" - args: - [ - "storage", - "rm", - "--recursive", - "gs://gcsfs-test-versioned-${_SHORT_BUILD_ID}", - ] - waitFor: - - "run-tests-on-vm" - - # Step 10: Delete the HNS GCS bucket. + # Delete all GCS buckets in parallel. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "delete-hns-bucket" - entrypoint: "gcloud" + id: "delete-buckets" + entrypoint: "bash" args: - ["storage", "rm", "--recursive", "gs://gcsfs-test-hns-${_SHORT_BUILD_ID}"] + - "-c" + - | + set -e + echo "--- Deleting test buckets in parallel ---" + gcloud storage rm --recursive gs://gcsfs-test-standard-${_SHORT_BUILD_ID} & + gcloud storage rm --recursive gs://gcsfs-test-versioned-${_SHORT_BUILD_ID} & + gcloud storage rm --recursive gs://gcsfs-test-hns-${_SHORT_BUILD_ID} & + wait waitFor: - - "run-tests-on-vm" + - "run-standard-tests" + - "run-zonal-tests" + - "run-hns-tests" timeout: "3600s" # 60 minutes diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index b0f6b8bd..740737c4 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -532,12 +532,9 @@ def test_move(gcs): assert not gcs.exists(fn) -@pytest.mark.parametrize("slash_from", ([False, True])) -def test_move_recursive(gcs, slash_from): +def test_move_recursive_no_slash(gcs): # See issue #489 dir_from = TEST_BUCKET + "/nested" - if slash_from: - dir_from += "/" dir_to = TEST_BUCKET + "/new_name" gcs.mv(dir_from, dir_to, recursive=True) @@ -545,6 +542,16 @@ def test_move_recursive(gcs, slash_from): assert gcs.ls(dir_to) == [dir_to + "/file1", dir_to + "/file2", dir_to + "/nested2"] +def test_move_recursive_with_slash(gcs): + # See issue #489 + dir_from = TEST_BUCKET + "/nested/" + dir_to = TEST_BUCKET + "/new_name_with_slash" + + gcs.mv(dir_from, dir_to, recursive=True) + assert not gcs.exists(dir_from.rstrip("/")) + assert gcs.ls(dir_to) == [dir_to + "/file1", dir_to + "/file2", dir_to + "/nested2"] + + def test_cat_file(gcs): fn = TEST_BUCKET + "/test/accounts.1.json" data = gcs.cat_file(fn)