feat(xtest): add download-artifact.sh scripts for published SDK packages

.github/workflows/xtest.yml

@@ -11,17 +11,17 @@ on:
       otdfctl-ref:
         required: false
         type: string
-        default: main
+        default: main latest
         description: "The branch or commit to use for otdfctl"
       js-ref:
         required: false
         type: string
-        default: main
+        default: main latest
         description: "The branch or commit to use for the web-sdk"
       java-ref:
         required: false
         type: string
-        default: main
+        default: main latest
         description: "The branch or commit to use for the java-sdk"
       focus-sdk:
         required: false
@@ -37,15 +37,15 @@ on:
       otdfctl-ref:
         required: false
         type: string
-        default: main
+        default: main latest
       js-ref:
         required: false
         type: string
-        default: main
+        default: main latest
       java-ref:
         required: false
         type: string
-        default: main
+        default: main latest
       focus-sdk:
         required: false
         type: string
@@ -54,6 +54,11 @@ on:
     - cron: "30 6 * * *" # 0630 UTC
     - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday)
     - cron: "0 18 * * 0" # 1800 UTC (Sunday)
+
+concurrency:
+  group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   resolve-versions:
     timeout-minutes: 10
@@ -69,7 +74,7 @@ jobs:
       java: ${{ steps.version-info.outputs.java-version-info }}
       js: ${{ steps.version-info.outputs.js-version-info }}
     env:
-      PLATFORM_REF: "${{ inputs.platform-ref || 'main lts' }}"
+      PLATFORM_REF: "${{ inputs.platform-ref }}"
       JS_REF: "${{ inputs.js-ref }}"
       OTDFCTL_REF: "${{ inputs.otdfctl-ref }}"
       JAVA_REF: "${{ inputs.java-ref }}"
@@ -91,13 +96,13 @@ jobs:
             echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV"
           elif [[ $CRON_MONDAY_WEDNESDAY == 'true' ]]; then
             echo "Running Monday/Wednesday tests"
-            echo "DEFAULT_TAGS=main lts" >> "$GITHUB_ENV"
+            echo "DEFAULT_TAGS=main" >> "$GITHUB_ENV"
           elif [[ $CRON_WEEKLY == 'true' ]]; then
             echo "Running weekly tests"
-            echo "DEFAULT_TAGS=main latest lts" >> "$GITHUB_ENV"
+            echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV"
           else
             echo "Running PR, Workflow Dispatch, or manual trigger"
-            echo "DEFAULT_TAGS=main" >> "$GITHUB_ENV"
+            echo "DEFAULT_TAGS=main latest" >> "$GITHUB_ENV"
           fi
         env:
           CRON_NIGHTLY: ${{ github.event.schedule == '30 6 * * *' }}
@@ -108,6 +113,7 @@ jobs:
           path: otdf-sdk
           persist-credentials: false
           repository: opentdf/tests
+          ref: ${{ github.head_ref || github.ref }}
           sparse-checkout: xtest/sdk
       - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
         with:
@@ -245,15 +251,17 @@ jobs:
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b
         with:
           python-version: "3.14"
-      - uses: bufbuild/buf-setup-action@2211e06e8cf26d628cda2eea15c95f8c42b080b3
+      - uses: bufbuild/buf-action@8f4a1456a0ab6a1eb80ba68e53832e6fcfacc16c # v1.3.0
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          setup_only: true
+          token: ${{ secrets.BUF_TOKEN }}
+          version: "1.56.0"
 
       - name: Set up JDK
         uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7
         with:
-          java-version: "11"
-          distribution: "adopt"
+          java-version: "17"
+          distribution: "temurin"
           server-id: github
 
       - name: Set up Node 22
@@ -269,12 +277,39 @@ jobs:
           sdk: js
           version-info: "${{ needs.resolve-versions.outputs.js }}"
 
+      - name: Cache npm
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }}
+          restore-keys: |
+            npm-${{ runner.os }}-
+
       ######## SETUP THE JS CLI #############
       - name: build and setup the web-sdk cli
         id: build-web-sdk
         run: |
-          make
+          # Use download-artifact.sh for released versions and dev (from dev release)
+          for row in $(echo "$JS_VERSION_INFO" | jq -c '.[]'); do
+            TAG=$(echo "$row" | jq -r '.tag')
+            HEAD=$(echo "$row" | jq -r '.head')
+            VERSION=$(echo "$row" | jq -r '.version // empty')
+
+            chmod +x ./download-artifact.sh
+            if [[ "$HEAD" == "true" ]]; then
+              echo "Downloading $TAG from dev release..."
+              ./download-artifact.sh dev "dist/$TAG"
+            elif [[ -n "$VERSION" ]]; then
+              echo "Using npm package for $TAG (version $VERSION)..."
+              ./download-artifact.sh "$VERSION" "dist/$TAG"
+            else
+              echo "Building $TAG from source (no version info)..."
+              VERSIONS="$TAG" make
+            fi
+          done
         working-directory: otdftests/xtest/sdk/js
+        env:
+          JS_VERSION_INFO: ${{ needs.resolve-versions.outputs.js }}
 
       ######## CHECKOUT GO CLI #############
       - name: Configure otdfctl
@@ -285,6 +320,16 @@ jobs:
           sdk: go
           version-info: "${{ needs.resolve-versions.outputs.go }}"
 
+      - name: Cache Go modules
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }}
+          restore-keys: |
+            go-${{ runner.os }}-
+
       - name: Resolve otdfctl heads
         id: resolve-otdfctl-heads
         run: |-
@@ -313,9 +358,28 @@ jobs:
 
       ######## SETUP THE GO CLI #############
       - name: Prepare go cli
-        run: |-
-          make
+        run: |
+          # Use download-artifact.sh for released versions, make for HEAD
+          for row in $(echo "$GO_VERSION_INFO" | jq -c '.[]'); do
+            TAG=$(echo "$row" | jq -r '.tag')
+            HEAD=$(echo "$row" | jq -r '.head')
+            VERSION=$(echo "$row" | jq -r '.version // empty')
+
+            if [[ "$HEAD" == "true" ]]; then
+              echo "Building $TAG from source (HEAD version)..."
+              VERSIONS="$TAG" make
+            elif [[ -n "$VERSION" ]]; then
+              echo "Using pre-built binary for $TAG (version $VERSION)..."
+              chmod +x ./download-artifact.sh
+              ./download-artifact.sh "$VERSION" "dist/$TAG"
+            else
+              echo "Building $TAG from source (no version info)..."
+              VERSIONS="$TAG" make
+            fi
+          done
         working-directory: otdftests/xtest/sdk/go
+        env:
+          GO_VERSION_INFO: ${{ needs.resolve-versions.outputs.go }}
 
       ####### CHECKOUT JAVA SDK ##############
 
@@ -326,6 +390,14 @@ jobs:
           sdk: java
           version-info: "${{ needs.resolve-versions.outputs.java }}"
 
+      - name: Cache Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }}
+          restore-keys: |
+            maven-${{ runner.os }}-
+
       - name: pre-release protocol buffers for java-sdk
         if: |
           (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
@@ -351,11 +423,27 @@ jobs:
       ####### SETUP JAVA CLI ##############
       - name: Prepare java cli
         run: |
-          make
-        env:
-          BUF_INPUT_HTTPS_USERNAME: opentdf-bot
-          BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }}
+          # Use download-artifact.sh for released versions and dev (from dev release)
+          for row in $(echo "$JAVA_VERSION_INFO" | jq -c '.[]'); do
+            TAG=$(echo "$row" | jq -r '.tag')
+            HEAD=$(echo "$row" | jq -r '.head')
+            VERSION=$(echo "$row" | jq -r '.version // empty')
+
+            chmod +x ./download-artifact.sh
+            if [[ "$HEAD" == "true" ]]; then
+              echo "Downloading $TAG from dev release..."
+              ./download-artifact.sh dev "dist/$TAG"
+            elif [[ -n "$VERSION" ]]; then
+              echo "Using optimized build for $TAG (version $VERSION)..."
+              ./download-artifact.sh "$VERSION" "dist/$TAG"
+            else
+              echo "Building $TAG from source (no version info)..."
+              VERSIONS="$TAG" make
+            fi
+          done
         working-directory: otdftests/xtest/sdk/java
+        env:
+          JAVA_VERSION_INFO: ${{ needs.resolve-versions.outputs.java }}
 
       ######## Configure test environment #############
       - name: Lookup current platform version
@@ -422,7 +510,7 @@ jobs:
       ######## RUN THE TESTS #############
       - name: Run legacy decryption tests
         run: |-
-          uv run pytest --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_legacy.py
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_legacy.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -431,7 +519,7 @@ jobs:
       - name: Run all standard xtests
         if: ${{ env.FOCUS_SDK == 'all' }}
         run: |-
-          uv run pytest --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html  --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v test_tdfs.py test_policytypes.py
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v test_tdfs.py test_policytypes.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -441,7 +529,7 @@ jobs:
       - name: Run xtests focusing on a specific SDK
         if: ${{ env.FOCUS_SDK != 'all' }}
         run: |-
-          uv run pytest --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html  --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_tdfs.py test_policytypes.py
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_tdfs.py test_policytypes.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -531,7 +619,7 @@ jobs:
       - name: Run attribute based configuration tests
         if: ${{ steps.multikas.outputs.supported == 'true' }}
         run: |-
-          uv run pytest --html=test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_abac.py
+          uv run pytest -n auto --dist loadscope --html=test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_abac.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -588,6 +676,48 @@ jobs:
               throw err; // Re-throw unexpected errors
             }
 
+  test-artifacts:
+    # Validate that published SDK artifacts can be downloaded
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af
+        with:
+          node-version: "22"
+
+      - uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7
+        with:
+          java-version: "17"
+          distribution: "temurin"
+
+      - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version: "1.24"
+
+      - name: Test Java artifact build
+        run: |
+          chmod +x xtest/sdk/java/download-artifact.sh
+          xtest/sdk/java/download-artifact.sh 0.12.0 /tmp/java-test
+          ls -la /tmp/java-test/
+          java -jar /tmp/java-test/cmdline.jar --version
+
+      - name: Test JS artifact download
+        run: |
+          chmod +x xtest/sdk/js/download-artifact.sh
+          xtest/sdk/js/download-artifact.sh 0.4.0 /tmp/js-test
+          ls -la /tmp/js-test/
+
+      - name: Test Go artifact download
+        run: |
+          chmod +x xtest/sdk/go/download-artifact.sh
+          xtest/sdk/go/download-artifact.sh 0.24.0 /tmp/go-test
+          ls -la /tmp/go-test/
+
   xtest:
     # Capstone job: always runs, evaluates matrix job aggregate result and fails if any xct variant failed
     runs-on: ubuntu-latest

AGENTS.md

@@ -0,0 +1,39 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+
+- `xtest/`: Cross-client compatibility test harness (Python + `pytest`), with fixtures in `xtest/fixtures/` and golden data in `xtest/golden/`.
+- `xtest/sdk/`: Helper scripts and Makefiles for checking out/building SDKs under test (e.g., `xtest/sdk/scripts/checkout-all.sh`, `cd xtest/sdk && make`).
+- `vulnerability/`: Playwright-based security regression tests (`vulnerability/tests/`).
+- `.github/workflows/`: CI workflows (lint/type-check, xtest matrix runs, vulnerability runs).
+
+## Build, Test, and Development Commands
+
+- Enter the dev environment: `devbox shell` (installs Python/JDK/Node per `devbox.json`).
+- Install xtest deps: `cd xtest && uv sync` (or `uv sync --extra dev` for dev tools)
+- Run xtest: `cd xtest && uv run pytest`
+  - Focus a subset: `uv run pytest --sdks "go js" --focus go` (see `xtest/conftest.py` options)
+  - HTML report: `uv run pytest --html tmp/test-report.html --self-contained-html`
+- Build SDK CLIs (after checkout): `cd xtest/sdk && make`
+- Run vulnerability tests: `cd vulnerability && npm ci && npm test` (requires a running platform; see `README.md` and `vulnerability/README.md`).
+
+## Coding Style & Naming Conventions
+
+- Python code in `xtest/` uses 4-space indentation, `snake_case`, and `pytest`-style fixtures.
+- CI enforces these checks in `xtest/`:
+  - `ruff check` and `ruff format --check`
+  - `pyright`
+  - Local equivalent: `cd xtest && uv sync --extra dev && uv run ruff check . && uv run ruff format --check . && uv run pyright`
+
+## Testing Guidelines
+
+- `pytest` tests live in `xtest/test_*.py`; add new fixtures under `xtest/fixtures/`.
+- Tests assume a platform backend is reachable (Docker + Keycloak). Use `xtest/test.env` as a template:
+  - `cd xtest && set -a && source test.env && set +a`
+
+## Commit & Pull Request Guidelines
+
+- Use semantic commit/PR titles (enforced by CI): `feat(xtest): ...`, `fix(vulnerability): ...`, `docs: ...` (types: `fix|feat|chore|docs`; scopes include `xtest`, `vulnerability`, `go`, `java`, `web`, `ci`).
+- DCO sign-off is required: `git commit -s -m "feat(xtest): ..."` (see `CONTRIBUTING.md`).
+- PRs should include a clear description, linked issue (if any), and relevant logs/screenshots for test failures; reviewers are defined in `CODEOWNERS`.
+

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

xtest/conftest.py

@@ -204,9 +204,14 @@ def pt_file(tmp_dir: Path, size: str) -> Path:
 
 
 @pytest.fixture(scope="package")
-def tmp_dir() -> Path:
-    """Create and return temporary directory for test files."""
-    dname = Path("tmp/")
+def tmp_dir(request: pytest.FixtureRequest) -> Path:
+    """Create worker-specific temporary directory for test files.
+
+    When running with pytest-xdist, each worker gets its own subdirectory
+    to prevent file collisions between parallel test processes.
+    """
+    worker_id = getattr(request.config, "workerinput", {}).get("workerid", "master")
+    dname = Path(f"tmp/{worker_id}/")
     dname.mkdir(parents=True, exist_ok=True)
     return dname
 

xtest/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
     "Pygments>=2.19.2",
     "pytest>=9.0.2",
     "pytest-html>=4.1.1",
+    "pytest-xdist>=3.6.1",
     "pytest-metadata>=3.1.1",
     "referencing>=0.37.0",
     "requests>=2.32.5",