Merge branch 'main' of github.com:NVIDIA/OpenShell into docs/structure-cleanup

Author: miyoungc

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -182,6 +182,8 @@ Component images (server, sandbox) can reach kubelet via two paths:
 
 **Local/external pull mode** (default local via `mise run cluster`): Local images are tagged to the configured local registry base (default `127.0.0.1:5000/openshell/*`), pushed to that registry, and pulled by k3s via `registries.yaml` mirror endpoint (typically `host.docker.internal:5000`). The `cluster` task pushes prebuilt local tags (`openshell/*:dev`, falling back to `localhost:5000/openshell/*:dev` or `127.0.0.1:5000/openshell/*:dev`).
 
+Gateway image builds now stage a partial Rust workspace from `deploy/docker/Dockerfile.images`. If cargo fails with a missing manifest under `/build/crates/...`, verify that every current gateway dependency crate (including `openshell-driver-kubernetes`) is copied into the staged workspace there.
+
 ```bash
 # Verify image refs currently used by openshell deployment
 openshell doctor exec -- kubectl -n openshell get statefulset openshell -o jsonpath="{.spec.template.spec.containers[*].image}"

.github/workflows/release-dev.yml

@@ -89,14 +89,70 @@ jobs:
               "${REGISTRY}/${component}:${{ github.sha }}"
           done
 
-  build-python-wheels:
-    name: Stage Python Wheels
+  build-python-wheels-linux:
+    name: Build Python Wheels (Linux ${{ matrix.arch }})
+    needs: [compute-versions]
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+            runner: build-amd64
+            artifact: linux-amd64
+            task: python:build:linux:amd64
+            output_path: target/wheels/linux-amd64/*.whl
+          - arch: arm64
+            runner: build-arm64
+            artifact: linux-arm64
+            task: python:build:linux:arm64
+            output_path: target/wheels/linux-arm64/*.whl
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    container:
+      image: ghcr.io/nvidia/openshell/ci:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --privileged
+    env:
+      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
+      OPENSHELL_IMAGE_TAG: dev
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Mark workspace safe for git
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Sync Python dependencies
+        run: uv sync
+
+      - name: Cache Rust target and registry
+        uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+        with:
+          shared-key: python-wheel-linux-${{ matrix.arch }}
+          cache-directories: .cache/sccache
+          cache-targets: "true"
+
+      - name: Build Python wheels
+        run: |
+          set -euo pipefail
+          OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }}
+          ls -la ${{ matrix.output_path }}
+
+      - name: Upload wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-wheels-${{ matrix.artifact }}
+          path: ${{ matrix.output_path }}
+          retention-days: 5
+
+  build-python-wheel-macos:
+    name: Build Python Wheel (macOS)
     needs: [compute-versions]
     runs-on: build-amd64
     timeout-minutes: 120
-    outputs:
-      wheel_version: ${{ needs.compute-versions.outputs.python_version }}
-      wheel_filenames: ${{ steps.filenames.outputs.wheel_filenames }}
     container:
       image: ghcr.io/nvidia/openshell/ci:latest
       credentials:
@@ -126,29 +182,24 @@ jobs:
       - name: Sync Python dependencies
         run: uv sync
 
-      - name: Build Python wheels
+      - name: Build Python wheel
         run: |
           set -euo pipefail
-          OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:multiarch
           OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos
           ls -la target/wheels/*.whl
 
-      - name: Capture wheel filenames
-        id: filenames
-        run: |
-          set -euo pipefail
-          WHEEL_FILENAMES=$(ls target/wheels/*.whl | xargs -n1 basename | paste -sd, -)
-          echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT"
-
       - name: Upload wheel artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: python-wheels
+          name: python-wheels-macos
           path: target/wheels/*.whl
           retention-days: 5
 
   # ---------------------------------------------------------------------------
   # Build CLI binaries (Linux musl — static, native on each arch)
+  #
+  # Builds run directly on the CI host (glibc Ubuntu).  Zig provides musl
+  # C/C++ toolchains for bundled-z3 and ring, and is also used as the linker.
   # ---------------------------------------------------------------------------
   build-cli-linux:
     name: Build CLI (Linux ${{ matrix.arch }})
@@ -159,9 +210,11 @@ jobs:
           - arch: amd64
             runner: build-amd64
             target: x86_64-unknown-linux-musl
+            zig_target: x86_64-linux-musl
           - arch: arm64
             runner: build-arm64
             target: aarch64-unknown-linux-musl
+            zig_target: aarch64-linux-musl
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 60
     container:
@@ -195,23 +248,44 @@ jobs:
           cache-directories: .cache/sccache
           cache-targets: "true"
 
-      - name: Install musl toolchain
+      - name: Add Rust musl target
+        run: mise x -- rustup target add ${{ matrix.target }}
+
+      - name: Set up zig musl wrappers
         run: |
           set -euo pipefail
-          apt-get update
-          apt-get install -y --no-install-recommends musl-tools
-          rm -rf /var/lib/apt/lists/*
+          ZIG="$(mise which zig)"
+          ZIG_TARGET="${{ matrix.zig_target }}"
+          mkdir -p /tmp/zig-musl
+
+          # cc-rs injects --target=<rust-triple> (for example
+          # aarch64-unknown-linux-musl), which zig does not parse.  Strip any
+          # caller-provided --target and use the wrapper's zig-native target.
+          for tool in cc c++; do
+            printf '#!/bin/bash\nargs=()\nfor arg in "$@"; do\n  case "$arg" in\n    --target=*) ;;\n    *) args+=("$arg") ;;\n  esac\ndone\nexec "%s" %s --target=%s "${args[@]}"\n' \
+              "$ZIG" "$tool" "$ZIG_TARGET" > "/tmp/zig-musl/${tool}"
+            chmod +x "/tmp/zig-musl/${tool}"
+          done
 
-      - name: Add Rust musl target
-        run: mise x -- rustup target add ${{ matrix.target }}
+          TARGET_ENV=$(echo "${{ matrix.target }}" | tr '-' '_')
+          TARGET_ENV_UPPER=${TARGET_ENV^^}
+
+          # Use zig for C/C++ compilation and final linking.
+          echo "CC_${TARGET_ENV}=/tmp/zig-musl/cc" >> "$GITHUB_ENV"
+          echo "CXX_${TARGET_ENV}=/tmp/zig-musl/c++" >> "$GITHUB_ENV"
+          echo "CARGO_TARGET_${TARGET_ENV_UPPER}_LINKER=/tmp/zig-musl/cc" >> "$GITHUB_ENV"
+
+          # Let zig own CRT/startfiles to avoid duplicate _start symbols.
+          echo "CARGO_TARGET_${TARGET_ENV_UPPER}_RUSTFLAGS=-Clink-self-contained=no" >> "$GITHUB_ENV"
+
+          # z3 built with zig c++ uses libc++ symbols (std::__1::*).
+          # Override z3-sys default (stdc++) so Rust links the matching runtime.
+          echo "CXXSTDLIB=c++" >> "$GITHUB_ENV"
 
       - name: Scope workspace to CLI crates
         run: |
           set -euo pipefail
-          # Remove workspace members that are not needed for openshell-cli.
-          # This avoids Cargo feature-unification pulling in aws-lc-sys (via
-          # russh in openshell-sandbox / openshell-server).
-          sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-cli", "crates/openshell-core", "crates/openshell-bootstrap", "crates/openshell-policy", "crates/openshell-providers", "crates/openshell-tui"]|' Cargo.toml
+          sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-cli", "crates/openshell-core", "crates/openshell-bootstrap", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-providers", "crates/openshell-tui"]|' Cargo.toml
 
       - name: Patch workspace version
         if: needs.compute-versions.outputs.cargo_version != ''
@@ -220,7 +294,7 @@ jobs:
           sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml
 
       - name: Build ${{ matrix.target }}
-        run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli
+        run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli --features bundled-z3
 
       - name: sccache stats
         if: always()
@@ -309,9 +383,11 @@ jobs:
   # ---------------------------------------------------------------------------
   release-dev:
     name: Release Dev
-    needs: [build-cli-linux, build-cli-macos, build-python-wheels]
+    needs: [compute-versions, build-cli-linux, build-cli-macos, build-python-wheels-linux, build-python-wheel-macos]
     runs-on: build-amd64
     timeout-minutes: 10
+    outputs:
+      wheel_filenames: ${{ steps.wheel_filenames.outputs.wheel_filenames }}
     steps:
       - uses: actions/checkout@v4
 
@@ -325,8 +401,17 @@ jobs:
       - name: Download wheel artifacts
         uses: actions/download-artifact@v4
         with:
-          name: python-wheels
+          pattern: python-wheels-*
           path: release/
+          merge-multiple: true
+
+      - name: Capture wheel filenames
+        id: wheel_filenames
+        run: |
+          set -euo pipefail
+          ls -la release/*.whl
+          WHEEL_FILENAMES=$(ls release/*.whl | xargs -n1 basename | sort | paste -sd, -)
+          echo "wheel_filenames=${WHEEL_FILENAMES}" >> "$GITHUB_OUTPUT"
 
       - name: Generate checksums
         run: |
@@ -338,7 +423,7 @@ jobs:
       - name: Prune stale wheel assets from dev release
         uses: actions/github-script@v7
         env:
-          WHEEL_VERSION: ${{ needs.build-python-wheels.outputs.wheel_version }}
+          WHEEL_VERSION: ${{ needs.compute-versions.outputs.python_version }}
         with:
           script: |
             const wheelVersion = process.env.WHEEL_VERSION;
@@ -416,7 +501,7 @@ jobs:
 
   trigger-wheel-publish:
     name: Trigger Wheel Publish
-    needs: [compute-versions, build-python-wheels, release-dev]
+    needs: [compute-versions, release-dev]
     runs-on: [self-hosted, nv]
     timeout-minutes: 10
     steps:
@@ -425,7 +510,7 @@ jobs:
           GITLAB_CI_TRIGGER_TOKEN: ${{ secrets.GITLAB_CI_TRIGGER_TOKEN }}
           GITLAB_CI_TRIGGER_URL: ${{ secrets.GITLAB_CI_TRIGGER_URL }}
           RELEASE_VERSION: ${{ needs.compute-versions.outputs.python_version }}
-          WHEEL_FILENAMES: ${{ needs.build-python-wheels.outputs.wheel_filenames }}
+          WHEEL_FILENAMES: ${{ needs.release-dev.outputs.wheel_filenames }}
         run: |
           set -euo pipefail
           if [ -z "${WHEEL_FILENAMES}" ]; then