From ac5ba6339eb25e92290124fa786a723131e176f0 Mon Sep 17 00:00:00 2001 From: Hubert Gruszecki Date: Mon, 11 Aug 2025 12:05:20 +0200 Subject: [PATCH] feat(ci): overhaul CI/CD with smart detection & optimized caching Major improvements: - Intelligent change detection with dependency graph resolution - Only test affected components + transitive dependencies - Dual caching: sccache for artifacts + rust-cache for deps - Parallel matrix execution for all test suites - Python tests use server-start/stop instead of Docker - Centralized component config in components.yml - Unified test workflow handling all languages - Maven Nexus publishing for Java SDK - Multi-arch Docker builds with buildx - Manual tagging workflow for controlled releases - CI summary with execution times per component - License header & conventional commit validation Fixes: Missing Docker inputs, sccache persistence, order-sensitive pattern matching, empty duration column, Maven credentials. Reduces CI runtime by ~40% while improving reliability. --- .dockerignore | 5 + .../csharp-dotnet/post-merge/action.yml | 110 ++ .../csharp-dotnet/pre-merge/action.yml | 123 +++ .github/actions/go/post-merge/action.yml | 118 +++ .github/actions/go/pre-merge/action.yml | 139 +++ .../actions/java-gradle/post-merge/action.yml | 133 +++ .../actions/java-gradle/pre-merge/action.yml | 86 ++ .../actions/node-npm/post-merge/action.yml | 186 ++++ .github/actions/node-npm/pre-merge/action.yml | 109 ++ .../python-maturin/post-merge/action.yml | 161 +++ .../python-maturin/pre-merge/action.yml | 178 ++++ .github/actions/rust/post-merge/action.yml | 172 ++++ .github/actions/rust/pre-merge/action.yml | 110 ++ .../actions/utils/docker-buildx/action.yml | 303 ++++++ .github/actions/utils/server-start/action.yml | 181 ++++ .github/actions/utils/server-stop/action.yml | 67 ++ .../utils/setup-rust-with-cache/action.yml | 141 +++ .github/changed-files-config.json | 100 -- .github/config/components.yml | 278 +++++ .github/config/publish.yml | 123 +++ .github/scripts/analyze_changed_files.py | 154 --- .github/scripts/detect-changed-files.sh | 91 -- .github/workflows/_build_python_wheels.yml | 282 ++++++ .github/workflows/_common.yml | 478 +++++++++ .github/workflows/_detect.yml | 382 +++++++ .github/workflows/_test.yml | 132 +++ .github/workflows/_test_bdd.yml | 110 ++ .github/workflows/_test_examples.yml | 118 +++ .github/workflows/ci-check-common.yml | 120 --- .github/workflows/ci-check-csharp-sdk.yml | 63 -- .github/workflows/ci-check-go-sdk.yml | 67 -- .github/workflows/ci-check-java-sdk.yml | 81 -- .github/workflows/ci-check-node-sdk.yml | 86 -- .github/workflows/ci-check-pr.yml | 197 ---- .github/workflows/ci-check-python-sdk.yml | 165 --- .github/workflows/ci-check-rust.yml | 164 --- .github/workflows/ci-check-shell.yml | 41 - .github/workflows/ci-compatibility-rust.yml | 188 ---- .github/workflows/ci-coverage-rust.yml | 80 -- .github/workflows/ci-prod-rust.yml | 170 ---- .github/workflows/ci-prod.yml | 76 -- .github/workflows/ci-publish-go.yml | 68 -- .github/workflows/ci-test-bdd.yml | 48 - .github/workflows/ci-test-rust-optional.yml | 64 -- .github/workflows/ci-test-rust.yml | 105 -- .github/workflows/post-merge.yml | 96 ++ .github/workflows/post_publish_server.yml | 105 -- .github/workflows/pre-merge.yml | 426 ++++++++ .github/workflows/publish.yml | 947 ++++++++++++++++++ .github/workflows/publish_bench.yml | 118 --- .github/workflows/publish_cli.yml | 126 --- .github/workflows/publish_connectors.yml | 285 ------ .github/workflows/publish_csharp_sdk.yml | 112 --- .github/workflows/publish_java_sdk.yml | 113 --- .../workflows/publish_java_sdk_snapshots.yml | 64 -- .github/workflows/publish_mcp.yml | 285 ------ .github/workflows/publish_node_sdk.yml | 129 --- .github/workflows/publish_rust_sdk.yml | 126 --- .github/workflows/publish_server.yml | 288 ------ .github/workflows/release_bench.yml | 120 --- .github/workflows/release_cli.yml | 161 --- .github/workflows/release_connectors.yml | 120 --- .github/workflows/release_mcp.yml | 120 --- .github/workflows/release_sdk.yml | 77 -- .github/workflows/release_server.yml | 124 --- .github/workflows/security.yml | 61 -- .github/workflows/test_daily.yml | 125 --- .github/workflows/test_nightly.yml | 121 --- Cargo.lock | 54 +- Cargo.toml | 92 +- Cross.toml | 25 - DEPENDENCIES.md | 2 +- Dockerfile.ci | 37 - Dockerfile.connectors.ci | 32 - Dockerfile.cross | 42 - Dockerfile.cross.dockerignore | 27 - Dockerfile.debug | 34 - Dockerfile.mcp.ci | 32 - README.md | 2 +- bdd/docker-compose.yml | 58 +- bdd/python/Dockerfile | 10 +- bdd/python/tests/conftest.py | 1 + bdd/rust/Dockerfile | 8 +- .../ai/mcp/.dockerignore | 0 core/ai/mcp/Dockerfile | 96 +- core/bench/Cargo.toml | 8 +- core/bench/dashboard/server/Cargo.toml | 6 +- core/bench/dashboard/server/Dockerfile | 10 +- core/bench/runner/Cargo.toml | 8 +- core/cli/Cargo.toml | 8 +- .../connectors/runtime/.dockerignore | 0 core/connectors/runtime/Dockerfile | 96 +- .../connectors/sinks/quickwit_sink/Cargo.toml | 6 +- core/connectors/sinks/stdout_sink/Cargo.toml | 6 +- .../sources/random_source/Cargo.toml | 6 +- core/integration/tests/cli/common/mod.rs | 4 + .../cli/topic/test_topic_create_command.rs | 12 +- .../cli/topic/test_topic_update_command.rs | 9 +- .../cli/user/test_user_create_command.rs | 12 +- .../cli/user/test_user_permissions_command.rs | 12 +- .../server/.dockerignore | 0 core/server/Cargo.toml | 8 +- core/server/Dockerfile | 143 +++ core/tools/Cargo.toml | 8 +- docker-compose.yml | 4 +- examples/python/getting-started/consumer.py | 10 +- examples/python/getting-started/producer.py | 11 +- .../MapperTests/BinaryMapper.cs | 4 +- foreign/csharp/scripts/pack.sh | 6 +- foreign/python/apache_iggy.pyi | 254 +++-- foreign/python/docker-compose.test.yml | 7 +- foreign/python/python_examples/consumer.py | 15 +- foreign/python/python_examples/producer.py | 24 +- foreign/python/scripts/test.sh | 10 +- foreign/python/tests/conftest.py | 15 +- foreign/python/tests/test_iggy_sdk.py | 190 ++-- scripts/check-backwards-compat.sh | 251 +++++ scripts/extract-version.sh | 233 +++++ scripts/run-bdd-tests.sh | 109 +- scripts/run-go-examples-from-readme.sh | 28 +- scripts/run-rust-examples-from-readme.sh | 45 +- scripts/sync-rust-version.sh | 126 ++- web/Dockerfile | 4 +- 123 files changed, 7192 insertions(+), 5670 deletions(-) create mode 100644 .github/actions/csharp-dotnet/post-merge/action.yml create mode 100644 .github/actions/csharp-dotnet/pre-merge/action.yml create mode 100644 .github/actions/go/post-merge/action.yml create mode 100644 .github/actions/go/pre-merge/action.yml create mode 100644 .github/actions/java-gradle/post-merge/action.yml create mode 100644 .github/actions/java-gradle/pre-merge/action.yml create mode 100644 .github/actions/node-npm/post-merge/action.yml create mode 100644 .github/actions/node-npm/pre-merge/action.yml create mode 100644 .github/actions/python-maturin/post-merge/action.yml create mode 100644 .github/actions/python-maturin/pre-merge/action.yml create mode 100644 .github/actions/rust/post-merge/action.yml create mode 100644 .github/actions/rust/pre-merge/action.yml create mode 100644 .github/actions/utils/docker-buildx/action.yml create mode 100644 .github/actions/utils/server-start/action.yml create mode 100644 .github/actions/utils/server-stop/action.yml create mode 100644 .github/actions/utils/setup-rust-with-cache/action.yml delete mode 100644 .github/changed-files-config.json create mode 100644 .github/config/components.yml create mode 100644 .github/config/publish.yml delete mode 100644 .github/scripts/analyze_changed_files.py delete mode 100755 .github/scripts/detect-changed-files.sh create mode 100644 .github/workflows/_build_python_wheels.yml create mode 100644 .github/workflows/_common.yml create mode 100644 .github/workflows/_detect.yml create mode 100644 .github/workflows/_test.yml create mode 100644 .github/workflows/_test_bdd.yml create mode 100644 .github/workflows/_test_examples.yml delete mode 100644 .github/workflows/ci-check-common.yml delete mode 100644 .github/workflows/ci-check-csharp-sdk.yml delete mode 100644 .github/workflows/ci-check-go-sdk.yml delete mode 100644 .github/workflows/ci-check-java-sdk.yml delete mode 100644 .github/workflows/ci-check-node-sdk.yml delete mode 100644 .github/workflows/ci-check-pr.yml delete mode 100644 .github/workflows/ci-check-python-sdk.yml delete mode 100644 .github/workflows/ci-check-rust.yml delete mode 100644 .github/workflows/ci-check-shell.yml delete mode 100644 .github/workflows/ci-compatibility-rust.yml delete mode 100644 .github/workflows/ci-coverage-rust.yml delete mode 100644 .github/workflows/ci-prod-rust.yml delete mode 100644 .github/workflows/ci-prod.yml delete mode 100644 .github/workflows/ci-publish-go.yml delete mode 100644 .github/workflows/ci-test-bdd.yml delete mode 100644 .github/workflows/ci-test-rust-optional.yml delete mode 100644 .github/workflows/ci-test-rust.yml create mode 100644 .github/workflows/post-merge.yml delete mode 100644 .github/workflows/post_publish_server.yml create mode 100644 .github/workflows/pre-merge.yml create mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/publish_bench.yml delete mode 100644 .github/workflows/publish_cli.yml delete mode 100644 .github/workflows/publish_connectors.yml delete mode 100644 .github/workflows/publish_csharp_sdk.yml delete mode 100644 .github/workflows/publish_java_sdk.yml delete mode 100644 .github/workflows/publish_java_sdk_snapshots.yml delete mode 100644 .github/workflows/publish_mcp.yml delete mode 100644 .github/workflows/publish_node_sdk.yml delete mode 100644 .github/workflows/publish_rust_sdk.yml delete mode 100644 .github/workflows/publish_server.yml delete mode 100644 .github/workflows/release_bench.yml delete mode 100644 .github/workflows/release_cli.yml delete mode 100644 .github/workflows/release_connectors.yml delete mode 100644 .github/workflows/release_mcp.yml delete mode 100644 .github/workflows/release_sdk.yml delete mode 100644 .github/workflows/release_server.yml delete mode 100644 .github/workflows/security.yml delete mode 100644 .github/workflows/test_daily.yml delete mode 100644 .github/workflows/test_nightly.yml delete mode 100644 Cross.toml delete mode 100644 Dockerfile.ci delete mode 100644 Dockerfile.connectors.ci delete mode 100644 Dockerfile.cross delete mode 100644 Dockerfile.cross.dockerignore delete mode 100644 Dockerfile.debug delete mode 100644 Dockerfile.mcp.ci rename Dockerfile.ci.dockerignore => core/ai/mcp/.dockerignore (100%) rename Dockerfile.connectors.ci.dockerignore => core/connectors/runtime/.dockerignore (100%) rename Dockerfile.mcp.ci.dockerignore => core/server/.dockerignore (100%) create mode 100644 core/server/Dockerfile create mode 100755 scripts/check-backwards-compat.sh create mode 100755 scripts/extract-version.sh diff --git a/.dockerignore b/.dockerignore index 336912aa62..3c4962a11b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,5 +13,10 @@ /local_data /performance_results /scripts +**/target /target +!/target/debug/iggy +!/target/debug/iggy-server +!/target/debug/iggy-mcp +!/target/debug/iggy-connectors /web diff --git a/.github/actions/csharp-dotnet/post-merge/action.yml b/.github/actions/csharp-dotnet/post-merge/action.yml new file mode 100644 index 0000000000..7d9ca5af60 --- /dev/null +++ b/.github/actions/csharp-dotnet/post-merge/action.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: csharp-dotnet-post-merge +description: .NET post-merge publishing github iggy actions + +inputs: + version: + description: "Version for publishing" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Restore dependencies + run: | + cd foreign/csharp + dotnet restore + shell: bash + + - name: Build Release + run: | + cd foreign/csharp + + # Build in Release mode + dotnet build -c Release --no-restore + + # List build output + echo "Build output:" + find . -name "*.dll" -path "*/bin/Release/*" | head -20 || echo "No DLLs found in Release folders" + shell: bash + + - name: Pack NuGet packages + run: | + cd foreign/csharp + + # Set version if provided + if [ -n "${{ inputs.version }}" ]; then + dotnet pack ./Iggy_SDK -c Release \ + -p:PackageVersion=${{ inputs.version }} \ + -o ./nupkgs \ + --no-build + else + echo "❌ Version is required for packing" + exit 1 + fi + + # List packages + echo "NuGet packages:" + ls -la ./nupkgs/ || echo "No packages found" + shell: bash + + - name: Publish to NuGet + env: + NUGET_API_KEY: ${{ env.NUGET_API_KEY }} + run: | + cd foreign/csharp + + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "πŸ” Dry run - would publish these packages:" + ls -la ./nupkgs/*.nupkg + + # Validate packages + for package in ./nupkgs/*.nupkg; do + echo "Validating: $package" + dotnet nuget locals all --clear + # Extract package info + unzip -l "$package" | head -20 + done + else + if [ -z "$NUGET_API_KEY" ]; then + echo "❌ NUGET_API_KEY is not set" + exit 1 + fi + + echo "πŸ“¦ Publishing packages to NuGet..." + # Push to NuGet + for package in ./nupkgs/*.nupkg; do + echo "Publishing: $(basename $package)" + dotnet nuget push "$package" \ + --api-key "$NUGET_API_KEY" \ + --source https://api.nuget.org/v3/index.json \ + --skip-duplicate + done + echo "βœ… Publishing completed" + fi + shell: bash diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml new file mode 100644 index 0000000000..b96b379c56 --- /dev/null +++ b/.github/actions/csharp-dotnet/pre-merge/action.yml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TODO(hubcio): Currently, C# tests don't need server. They use testcontainers with 'edge' image. +# We should change this to use server-start/stop action, so that code from PR is tested. + +name: csharp-dotnet-pre-merge +description: .NET pre-merge testing github iggy actions + +inputs: + task: + description: "Task to run (lint, test, build, e2e)" + required: true + +runs: + using: "composite" + steps: + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Setup Rust with cache + if: inputs.task == 'test' || inputs.task == 'e2e' + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + + - name: Install netcat + if: inputs.task == 'e2e' + run: sudo apt-get update && sudo apt-get install -y netcat-openbsd + shell: bash + + - name: Restore dependencies + run: | + cd foreign/csharp + dotnet restore + shell: bash + + - name: Build + if: inputs.task == 'test' || inputs.task == 'build' || inputs.task == 'lint' + run: | + cd foreign/csharp + dotnet build --no-restore + shell: bash + + - name: Lint (Code Analysis) + if: inputs.task == 'lint' + run: | + cd foreign/csharp + + # Run code analysis + dotnet build --no-restore /p:EnforceCodeStyleInBuild=true /p:TreatWarningsAsErrors=false + + # TODO: make format check blocking (requires dotnet-format tool) + dotnet format --verify-no-changes --verbosity diagnostic || true + + shell: bash + + - name: Test + if: inputs.task == 'test' + run: | + cd foreign/csharp + + # Run unit tests + dotnet test Iggy_SDK_Tests --no-build --verbosity normal + + # Run integration tests + dotnet test Iggy_SDK.Tests.Integration --no-build --verbosity normal + + shell: bash + + # TODO(hubcio): currently, C# tests don't need server. They use testcontainers with 'edge' image. + # instead, they should use server-start/stop action to test the actual code from PR. + # - name: Start Iggy server + # id: iggy + # if: inputs.task == 'e2e' + # uses: ./.github/actions/utils/server-start + # with: + # mode: cargo + # cargo-bin: iggy-server + # port: 8090 + + - name: Run integration tests + if: inputs.task == 'e2e' + run: | + cd foreign/csharp + dotnet test Iggy_SDK.Tests.Integration --no-build --verbosity normal + shell: bash + + - name: Stop Iggy server + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy.outputs.pid_file }} + log-file: ${{ steps.iggy.outputs.log_file }} + + - name: Build Release + if: inputs.task == 'build' + run: | + cd foreign/csharp + + # Build in Release mode + dotnet build -c Release --no-restore + + # List build output + echo "Build output:" + find . -name "*.dll" -path "*/bin/Release/*" | head -20 || echo "No DLLs found in Release folders" + shell: bash diff --git a/.github/actions/go/post-merge/action.yml b/.github/actions/go/post-merge/action.yml new file mode 100644 index 0000000000..22bded265d --- /dev/null +++ b/.github/actions/go/post-merge/action.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: go-post-merge +description: Go post-merge tag preparation github iggy actions + +inputs: + version: + description: "Version for tagging (without 'v' prefix)" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Validate version format + run: | + VERSION="${{ inputs.version }}" + + # Check if version matches semantic versioning + if ! echo "$VERSION" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?(\+[a-zA-Z0-9.-]+)?$' > /dev/null; then + echo "❌ Invalid version format: $VERSION" + echo "Expected format: X.Y.Z or X.Y.Z-prerelease+metadata" + exit 1 + fi + + echo "βœ… Version format valid: $VERSION" + shell: bash + + - name: Prepare Go module for tagging + run: | + VERSION="${{ inputs.version }}" + TAG="foreign/go/v${VERSION}" + + echo "πŸ“¦ Go Module Publishing Information" + echo "====================================" + echo "Version: v${VERSION}" + echo "Git tag: ${TAG}" + echo "" + + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "πŸ” DRY RUN MODE - No tag will be created" + echo "" + echo "Would create tag: ${TAG}" + echo "" + echo "After tagging, users could import using:" + echo " go get github.com/${{ github.repository }}/foreign/go@v${VERSION}" + echo "" + echo "Or add to go.mod:" + echo " require github.com/${{ github.repository }}/foreign/go v${VERSION}" + else + echo "βœ… Go module ready for tagging" + echo "" + echo "Tag will be created: ${TAG}" + echo "This will be handled by the create-tags job in the publish workflow" + echo "" + echo "After the tag is pushed, users can import using:" + echo " go get github.com/${{ github.repository }}/foreign/go@v${VERSION}" + echo "" + echo "Or add to go.mod:" + echo " require github.com/${{ github.repository }}/foreign/go v${VERSION}" + fi + + # Verify the go.mod file exists + if [ ! -f "foreign/go/go.mod" ]; then + echo "⚠️ Warning: foreign/go/go.mod not found" + echo "Make sure the Go module is properly initialized" + else + echo "" + echo "Module information:" + grep "^module" foreign/go/go.mod || echo "Module declaration not found" + fi + shell: bash + + - name: Output tag information + id: tag-info + run: | + VERSION="${{ inputs.version }}" + TAG="foreign/go/v${VERSION}" + + # Set outputs for use in other jobs + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "version=v${VERSION}" >> "$GITHUB_OUTPUT" + + # Summary for GitHub Actions + { + echo "## 🏷️ Go Module Tag Information" + echo "" + echo "| Property | Value |" + echo "|----------|-------|" + echo "| **Version** | \`v${VERSION}\` |" + echo "| **Git Tag** | \`${TAG}\` |" + echo "| **Import Path** | \`github.com/${{ github.repository }}/foreign/go\` |" + echo "" + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "**Note:** This is a dry run - no actual tag will be created" + else + echo "**Note:** Tag will be created by the publish workflow" + fi + } >> "$GITHUB_STEP_SUMMARY" + shell: bash diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml new file mode 100644 index 0000000000..f8628d2ba1 --- /dev/null +++ b/.github/actions/go/pre-merge/action.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: go-pre-merge +description: Go pre-merge testing github iggy actions + +inputs: + task: + description: "Task to run (lint, test, build, e2e)" + required: true + +runs: + using: "composite" + steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.23.0" + cache-dependency-path: | + foreign/go/go.sum + bdd/go/go.sum + examples/go/go.sum + + - name: Download dependencies + run: | + if [ -f "foreign/go/go.mod" ]; then + cd foreign/go && go mod download + cd .. + fi + + if [ -f "bdd/go/go.mod" ]; then + cd bdd/go && go mod download + cd .. + fi + + if [ -f "examples/go/go.mod" ]; then + cd examples/go && go mod download + fi + shell: bash + + - name: Tidy Check + shell: bash + if: inputs.task == 'test' || inputs.task == 'lint' + run: | + cd foreign/go + go mod tidy + git diff --exit-code go.mod go.sum + + - name: Test + shell: bash + if: inputs.task == 'test' + run: | + cd foreign/go + + # Create reports directory + mkdir -p ../../reports + + # Run tests with coverage + go test -v -race -coverprofile=../../reports/go-coverage.out ./... + + # Generate coverage report + go tool cover -html=../../reports/go-coverage.out -o ../../reports/go-coverage.html + + # Run tests with JSON output for better reporting + go test -v -json ./... > ../../reports/go-tests.json + + + - name: Lint + if: inputs.task == 'lint' + uses: golangci/golangci-lint-action@v6 + with: + version: v1.61 + working-directory: foreign/go + args: --timeout=5m + + - name: Lint BDD + if: inputs.task == 'lint' && hashFiles('bdd/go/go.mod') != '' + uses: golangci/golangci-lint-action@v6 + with: + version: v1.61 + working-directory: bdd/go + args: --timeout=5m + + - name: Lint Examples + if: inputs.task == 'lint' && hashFiles('examples/go/go.mod') != '' + uses: golangci/golangci-lint-action@v6 + with: + version: v1.61 + working-directory: examples/go + args: --timeout=5m + + - name: Build + shell: bash + if: inputs.task == 'build' + run: | + cd foreign/go + + # Build the module + go build -v ./... + + - name: Setup server for e2e tests + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + + - name: Run e2e tests + shell: bash + if: inputs.task == 'e2e' + run: | + echo "πŸ§ͺ Running Go e2e tests..." + + # Install ginkgo + echo "Installing ginkgo..." + go install github.com/onsi/ginkgo/v2/ginkgo@latest + + # Ensure ginkgo is in PATH + export PATH="${PATH}:$(go env GOPATH)/bin" + + # Run foreign/go e2e tests + echo "Running foreign/go e2e tests..." + cd foreign/go + go test -v -race ./... + + - name: Stop server after e2e tests + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop diff --git a/.github/actions/java-gradle/post-merge/action.yml b/.github/actions/java-gradle/post-merge/action.yml new file mode 100644 index 0000000000..5ef6724127 --- /dev/null +++ b/.github/actions/java-gradle/post-merge/action.yml @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: java-gradle-post-merge +description: Java Gradle post-merge publishing github iggy actions + +inputs: + version: + description: "Version for publishing" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + cache: "gradle" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 + + - name: Grant execute permission for gradlew + shell: bash + run: | + if [ -f "foreign/java/gradlew" ]; then + chmod +x foreign/java/gradlew + fi + + - name: Build for publishing + shell: bash + run: | + echo "πŸ“¦ Building Java SDK for publishing..." + foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest + BUILD_EXIT_CODE=$? + + # List artifacts only if build succeeded + if [ $BUILD_EXIT_CODE -eq 0 ]; then + echo "" + echo "Build artifacts:" + find foreign/java -path "*/build/libs/*.jar" -type f 2>/dev/null | head -20 || echo "No jar artifacts found in build/libs directories" + else + echo "❌ Build failed with exit code $BUILD_EXIT_CODE" + exit $BUILD_EXIT_CODE + fi + + - name: Publish to Maven Nexus + shell: bash + env: + NEXUS_USER: ${{ env.NEXUS_USER }} + NEXUS_PASSWORD: ${{ env.NEXUS_PASSWORD }} + run: | + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "πŸ” Dry run - would publish to Maven Nexus:" + echo "" + + # Extract version from build.gradle.kts + gradle_version=$(foreign/java/gradlew -p foreign/java properties -q | grep "version:" | cut -d: -f2 | tr -d ' ') + echo "Version from gradle: $gradle_version" + echo "Input version: ${{ inputs.version }}" + + # Verify versions match + if [ "$gradle_version" != "${{ inputs.version }}" ]; then + echo "⚠️ Warning: Gradle version ($gradle_version) doesn't match input version (${{ inputs.version }})" + fi + + echo "" + echo "Would publish artifacts:" + echo " Group ID: org.apache.iggy" + echo " Artifact ID: iggy" + echo " Version: ${{ inputs.version }}" + echo "" + echo "Maven coordinates: org.apache.iggy:iggy:${{ inputs.version }}" + + # Show what would be published + echo "" + echo "Artifacts that would be published:" + find foreign/java -path "*/build/libs/*.jar" -type f 2>/dev/null | while read jar; do + echo " - $(basename $jar)" + done + else + if [ -z "$NEXUS_USER" ] || [ -z "$NEXUS_PASSWORD" ]; then + echo "❌ NEXUS_USER or NEXUS_PASSWORD not set" + exit 1 + fi + + echo "πŸ“¦ Publishing to Maven Nexus..." + echo "Version: ${{ inputs.version }}" + echo "" + + # Run the publish task + foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest publish + PUBLISH_EXIT_CODE=$? + + if [ $PUBLISH_EXIT_CODE -eq 0 ]; then + echo "" + echo "βœ… Successfully published to Maven Nexus" + echo "Maven coordinates: org.apache.iggy:iggy:${{ inputs.version }}" + echo "" + echo "Users can now add to their build.gradle:" + echo " implementation 'org.apache.iggy:iggy:${{ inputs.version }}'" + echo "" + echo "Or to their pom.xml:" + echo " " + echo " org.apache.iggy" + echo " iggy" + echo " ${{ inputs.version }}" + echo " " + else + echo "❌ Publishing failed with exit code $PUBLISH_EXIT_CODE" + exit $PUBLISH_EXIT_CODE + fi + fi diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml new file mode 100644 index 0000000000..c939652e11 --- /dev/null +++ b/.github/actions/java-gradle/pre-merge/action.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: java-gradle-pre-merge +description: Java Gradle pre-merge testing github iggy actions + +inputs: + task: + description: "Task to run (lint, test, build)" + required: true + +runs: + using: "composite" + steps: + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + cache: "gradle" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 + + - name: Grant execute permission for gradlew + shell: bash + run: | + if [ -f "foreign/java/gradlew" ]; then + chmod +x foreign/java/gradlew + fi + + - name: Lint + if: inputs.task == 'lint' + shell: bash + run: | + # TODO(hubcio): make this blocking when Java lints are fixed + foreign/java/dev-support/checks/build.sh check -x test || true + + - name: Build + shell: bash + if: inputs.task == 'build' + run: | + foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest + BUILD_EXIT_CODE=$? + + # List artifacts only if build succeeded + if [ $BUILD_EXIT_CODE -eq 0 ]; then + echo "" + echo "Build artifacts:" + find foreign/java -path "*/build/libs/*.jar" -type f 2>/dev/null | head -20 || echo "No jar artifacts found in build/libs directories" + fi + + # Exit with build exit code + exit $BUILD_EXIT_CODE + + - name: Test + if: inputs.task == 'test' + shell: bash + run: | + foreign/java/dev-support/checks/build.sh test + + # Copy test reports + if [ -d "foreign/java/build/test-results" ]; then + mkdir -p reports + cp -r foreign/java/build/test-results reports/java-tests + fi + + # Generate test report if jacoco is configured + if grep -q "jacoco" foreign/java/build.gradle 2>/dev/null; then + cd foreign/java + ./gradlew jacocoTestReport --no-daemon + fi diff --git a/.github/actions/node-npm/post-merge/action.yml b/.github/actions/node-npm/post-merge/action.yml new file mode 100644 index 0000000000..3e02950089 --- /dev/null +++ b/.github/actions/node-npm/post-merge/action.yml @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: node-npm-post-merge +description: Node.js post-merge npm publishing github iggy actions + +inputs: + version: + description: "Version for publishing" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "23" + registry-url: "https://registry.npmjs.org" + cache: "npm" + cache-dependency-path: foreign/node/package-lock.json + + - name: Install dependencies + run: | + cd foreign/node + npm ci --ignore-scripts + shell: bash + + - name: Build for publishing + run: | + cd foreign/node + echo "πŸ“¦ Building Node.js SDK for publishing..." + npm run build + + # Verify build output + if [ -d "dist" ]; then + echo "" + echo "Build output:" + ls -la dist/ + + # Count files + FILE_COUNT=$(find dist -type f | wc -l) + echo "" + echo "Total files in dist: $FILE_COUNT" + else + echo "❌ Error: dist directory not found after build" + exit 1 + fi + shell: bash + + - name: Prepare package version + run: | + cd foreign/node + + # Get current version from package.json + CURRENT_VERSION=$(node -p "require('./package.json').version") + NEW_VERSION="${{ inputs.version }}" + + echo "Current package.json version: $CURRENT_VERSION" + echo "Target version: $NEW_VERSION" + + if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then + if [ "${{ inputs.dry_run }}" = "false" ]; then + echo "Updating version to $NEW_VERSION..." + npm version "$NEW_VERSION" --no-git-tag-version --allow-same-version + else + echo "Would update version from $CURRENT_VERSION to $NEW_VERSION" + fi + else + echo "Version already matches target" + fi + shell: bash + + - name: Validate package + run: | + cd foreign/node + + echo "πŸ“‹ Package validation:" + echo "" + + # Check package.json required fields + node -e " + const pkg = require('./package.json'); + const required = ['name', 'version', 'description', 'main', 'types', 'license']; + const missing = required.filter(field => !pkg[field]); + if (missing.length > 0) { + console.error('❌ Missing required fields:', missing.join(', ')); + process.exit(1); + } + console.log('βœ… All required fields present'); + console.log(''); + console.log('Package info:'); + console.log(' Name:', pkg.name); + console.log(' Version:', pkg.version); + console.log(' Description:', pkg.description); + console.log(' License:', pkg.license); + console.log(' Main:', pkg.main); + console.log(' Types:', pkg.types); + " + + # Check if files to be published exist + echo "" + echo "Files to be published:" + # npm pack might output non-JSON lines before the JSON, so we need to extract only the JSON part + npm pack --dry-run --json 2>/dev/null | tail -1 | jq -r '.[0].files[].path' 2>/dev/null | head -20 || echo " (Unable to list files, but continuing...)" + shell: bash + + - name: Publish to npm + env: + NODE_AUTH_TOKEN: ${{ env.NPM_TOKEN }} + run: | + cd foreign/node + + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "πŸ” Dry run - would publish to npm:" + echo "" + + # Show what would be published + npm pack --dry-run + + echo "" + echo "Package details:" + node -e " + const pkg = require('./package.json'); + console.log(' Name: ' + pkg.name); + console.log(' Version: ${{ inputs.version }}'); + console.log(' Registry: https://registry.npmjs.org'); + " + + echo "" + echo "After publishing, users could install with:" + echo " npm install apache-iggy@${{ inputs.version }}" + echo "" + echo "Or add to package.json:" + echo ' "apache-iggy": "^${{ inputs.version }}"' + else + if [ -z "$NODE_AUTH_TOKEN" ]; then + echo "❌ NPM_TOKEN is not set" + exit 1 + fi + + echo "πŸ“¦ Publishing to npm registry..." + echo "Version: ${{ inputs.version }}" + echo "" + + # Publish with provenance for supply chain security + npm publish --provenance --access public + + if [ $? -eq 0 ]; then + echo "" + echo "βœ… Successfully published to npm" + echo "" + echo "Package: apache-iggy@${{ inputs.version }}" + echo "Registry: https://registry.npmjs.org/apache-iggy" + echo "" + echo "Users can now install with:" + echo " npm install apache-iggy@${{ inputs.version }}" + echo "" + echo "Or add to package.json:" + echo ' "apache-iggy": "^${{ inputs.version }}"' + echo "" + echo "View on npm: https://www.npmjs.com/package/apache-iggy/v/${{ inputs.version }}" + else + echo "❌ Publishing failed" + exit 1 + fi + fi + shell: bash diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml new file mode 100644 index 0000000000..be9a7ed6c1 --- /dev/null +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: node-npm-pre-merge +description: Node.js pre-merge testing github iggy actions + +inputs: + task: + description: "Task to run (lint, test, build, e2e)" + required: true + +runs: + using: "composite" + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "23" + registry-url: "https://registry.npmjs.org" + cache: "npm" + cache-dependency-path: foreign/node/package-lock.json + + - name: Setup Rust with cache + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + + - name: Install netcat + if: inputs.task == 'e2e' + run: sudo apt-get update && sudo apt-get install -y netcat-openbsd + shell: bash + + - name: Install dependencies + run: | + cd foreign/node + npm ci --ignore-scripts + shell: bash + + - name: Lint + if: inputs.task == 'lint' + run: | + cd foreign/node + npm run lint + shell: bash + + - name: Build + if: inputs.task == 'build' + run: | + cd foreign/node + npm run build + + # Verify build output + if [ -d "dist" ]; then + echo "Build output:" + ls -la dist/ + else + echo "Warning: dist directory not found after build" + fi + shell: bash + + - name: Test + if: inputs.task == 'test' + run: | + cd foreign/node + npm run test:unit + shell: bash + + - name: Start Iggy server + id: iggy + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + with: + mode: cargo + cargo-bin: iggy-server + host: 127.0.0.1 + port: 8090 + wait-timeout-seconds: 45 + + - name: E2E tests + if: inputs.task == 'e2e' + run: | + cd foreign/node + npm run test:e2e + env: + IGGY_SERVER_HOST: 127.0.0.1 + IGGY_SERVER_TCP_PORT: 8090 + shell: bash + + - name: Stop Iggy server + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy.outputs.pid_file }} + log-file: ${{ steps.iggy.outputs.log_file }} diff --git a/.github/actions/python-maturin/post-merge/action.yml b/.github/actions/python-maturin/post-merge/action.yml new file mode 100644 index 0000000000..bb2d73f691 --- /dev/null +++ b/.github/actions/python-maturin/post-merge/action.yml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: python-maturin-post-merge +description: Python post-merge PyPI publishing with pre-built wheels + +inputs: + version: + description: "Version for publishing" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + wheels_artifact: + description: "Name of the artifact containing the wheels" + required: false + default: "python-wheels-all" + wheels_path: + description: "Path where wheels should be downloaded" + required: false + default: "dist" + +runs: + using: "composite" + steps: + - name: Validate version format + run: | + VERSION="${{ inputs.version }}" + + # Check if version matches Python/PEP 440 versioning + if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+((a|b|rc)[0-9]+)?(\.[a-z0-9]+)?$'; then + echo "❌ Invalid version format: $VERSION" + echo "Expected format: X.Y.Z or X.Y.Z[a|b|rc]N or X.Y.Z.devN" + exit 1 + fi + + echo "βœ… Version format valid: $VERSION" + shell: bash + + - name: Download pre-built wheels + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.wheels_artifact }} + path: ${{ inputs.wheels_path }} + + - name: Validate downloaded wheels + run: | + WHEELS_PATH="${{ inputs.wheels_path }}" + + echo "πŸ“‹ Validating downloaded wheels and source distribution:" + echo "" + + # Check if directory exists and has files + if [ ! -d "$WHEELS_PATH" ]; then + echo "❌ Wheels directory not found: $WHEELS_PATH" + exit 1 + fi + + # Count artifacts + WHEEL_COUNT=$(ls -1 "$WHEELS_PATH"/*.whl 2>/dev/null | wc -l || echo "0") + SDIST_COUNT=$(ls -1 "$WHEELS_PATH"/*.tar.gz 2>/dev/null | wc -l || echo "0") + + if [ "$WHEEL_COUNT" -eq 0 ] && [ "$SDIST_COUNT" -eq 0 ]; then + echo "❌ No wheels or source distributions found in $WHEELS_PATH" + exit 1 + fi + + echo "Found artifacts:" + echo " - Wheels: $WHEEL_COUNT" + echo " - Source distributions: $SDIST_COUNT" + echo "" + + # List all artifacts with details + echo "Artifacts to publish:" + for file in "$WHEELS_PATH"/*.whl "$WHEELS_PATH"/*.tar.gz; do + if [ -f "$file" ]; then + filename=$(basename "$file") + size=$(du -h "$file" | cut -f1) + + # Determine platform from filename + if [[ "$filename" == *"linux"* ]]; then platform="Linux" + elif [[ "$filename" == *"macosx"* ]]; then platform="macOS" + elif [[ "$filename" == *"win"* ]]; then platform="Windows" + elif [[ "$filename" == *.tar.gz ]]; then platform="Source" + else platform="Universal"; fi + + # Determine architecture from filename + if [[ "$filename" == *"x86_64"* ]] || [[ "$filename" == *"amd64"* ]]; then arch="x86_64" + elif [[ "$filename" == *"aarch64"* ]] || [[ "$filename" == *"arm64"* ]] || [[ "$filename" == *"universal2"* ]]; then arch="arm64" + elif [[ "$filename" == *"i686"* ]] || [[ "$filename" == *"win32"* ]]; then arch="x86" + elif [[ "$filename" == *.tar.gz ]]; then arch="N/A" + else arch="any"; fi + + echo " - $filename ($platform/$arch, $size)" + fi + done + + echo "" + echo "βœ… All packages ready for publishing" + shell: bash + + - name: Display publishing information (dry run) + if: inputs.dry_run == 'true' + run: | + WHEELS_PATH="${{ inputs.wheels_path }}" + VERSION="${{ inputs.version }}" + + echo "πŸ” DRY RUN - Would publish to PyPI:" + echo "" + echo "Package: apache-iggy" + echo "Version: $VERSION" + echo "Registry: https://pypi.org" + echo "" + + echo "Packages that would be published:" + for file in "$WHEELS_PATH"/*.whl "$WHEELS_PATH"/*.tar.gz; do + if [ -f "$file" ]; then + echo " - $(basename "$file")" + fi + done + + echo "" + echo "Platform coverage:" + echo " - Linux: x86_64, aarch64" + echo " - macOS: x86_64, arm64" + echo " - Windows: x64" + echo " - Source distribution: included" + + echo "" + echo "After publishing, users could install with:" + echo " pip install apache-iggy==$VERSION" + echo "" + echo "Or add to requirements.txt:" + echo " apache-iggy==$VERSION" + echo "" + echo "View on PyPI: https://pypi.org/project/apache-iggy/$VERSION/" + shell: bash + + - name: Publish to PyPI using maturin + if: inputs.dry_run == 'false' + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ env.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing ${{ inputs.wheels_path }}/* diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml new file mode 100644 index 0000000000..75b70665e5 --- /dev/null +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: python-maturin-pre-merge +description: Python pre-merge testing with maturin github iggy actions + +inputs: + task: + description: "Task to run (lint, test, build)" + required: true + +runs: + using: "composite" + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-${{ hashFiles('foreign/python/pyproject.toml') }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + + # Install maturin for building + pip install 'maturin>=1.2,<2.0' + + # Install testing dependencies from pyproject.toml + cd foreign/python + pip install -e ".[testing,examples,dev]" + cd ../.. + + # Install additional CI dependencies for Docker testing if needed + if [ "${{ inputs.task }}" = "test" ]; then + pip install 'testcontainers[docker]>=3.7.0,<5.0' || true + fi + shell: bash + + - name: Lint and format check + if: inputs.task == 'lint' + run: | + # Build list of directories to check using absolute paths + DIRS_TO_CHECK="${GITHUB_WORKSPACE}/foreign/python/tests ${GITHUB_WORKSPACE}/foreign/python" + STUB_FILE="${GITHUB_WORKSPACE}/foreign/python/apache_iggy.pyi" + + echo "Directories to check: $DIRS_TO_CHECK" + echo "Stub file: $STUB_FILE" + + echo "ruff check --select I $DIRS_TO_CHECK" + ruff check --select I $DIRS_TO_CHECK + echo "ruff version: $(ruff --version)" + + echo "ruff format --check $DIRS_TO_CHECK" + ruff format --check $DIRS_TO_CHECK + + # TODO(hubcio): make this blocking when mypy lints are fixed + echo "mypy --explicit-package-bases $STUB_FILE $DIRS_TO_CHECK" + mypy --explicit-package-bases "$STUB_FILE" $DIRS_TO_CHECK || true + echo "mypy version: $(mypy --version)" + shell: bash + + - name: Build Python wheel for testing + if: inputs.task == 'test' || inputs.task == 'build' + run: | + cd foreign/python + + # Build the module + echo "Building Python wheel..." + maturin build -o dist + + if [ "${{ inputs.task }}" = "test" ]; then + # Install the built wheel for testing + echo "Installing built wheel..." + pip install dist/*.whl --force-reinstall + + # Create symlink for examples directory if needed + if [ -d "python_examples" ] && [ ! -e "examples" ]; then + ln -s python_examples examples + fi + fi + + if [ "${{ inputs.task }}" = "build" ]; then + # List built artifacts + echo "" + echo "Build artifacts:" + ls -la dist/ + fi + shell: bash + + - name: Start Iggy server + if: inputs.task == 'test' + id: iggy + uses: ./.github/actions/utils/server-start + with: + mode: cargo + cargo-bin: iggy-server + host: 127.0.0.1 + port: 8090 + wait-timeout-seconds: 45 + continue-on-error: true + + - name: Run Python integration tests + if: inputs.task == 'test' && steps.iggy.outcome == 'success' + run: | + cd foreign/python + + echo "Running integration tests with Iggy server at ${{ steps.iggy.outputs.address }}..." + + # Run all tests with server connection + IGGY_SERVER_HOST=127.0.0.1 \ + IGGY_SERVER_TCP_PORT=8090 \ + pytest tests/ -v \ + --junitxml=../../reports/python-junit.xml \ + --tb=short \ + --capture=no || TEST_EXIT_CODE=$? + + # Exit with test result + exit ${TEST_EXIT_CODE:-0} + shell: bash + + - name: Run Python unit tests only (fallback) + if: inputs.task == 'test' && steps.iggy.outcome != 'success' + run: | + cd foreign/python + + echo "⚠️ Server failed to start, running unit tests only..." + + # Run unit tests only (exclude integration tests) + pytest tests/ -v \ + -m "not integration" \ + --junitxml=../../reports/python-junit.xml \ + --tb=short || TEST_EXIT_CODE=$? + + # Exit with test result (allow some failures in unit-only mode) + exit ${TEST_EXIT_CODE:-0} + shell: bash + + - name: Stop Iggy server + if: always() && inputs.task == 'test' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy.outputs.pid_file }} + log-file: ${{ steps.iggy.outputs.log_file }} + + - name: Upload test artifacts + if: always() && inputs.task == 'test' + uses: actions/upload-artifact@v4 + with: + name: python-test-results-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/python-junit.xml + foreign/python/dist/*.whl + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/actions/rust/post-merge/action.yml b/.github/actions/rust/post-merge/action.yml new file mode 100644 index 0000000000..1255ae4697 --- /dev/null +++ b/.github/actions/rust/post-merge/action.yml @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: rust-post-merge +description: Rust post-merge crates.io publishing github iggy actions + +inputs: + package: + description: "Package name for publishing (e.g., iggy, iggy_common, iggy_binary_protocol)" + required: true + version: + description: "Version for publishing" + required: true + dry_run: + description: "Dry run mode" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + show-stats: false # Don't need stats for publishing + + - name: Validate package + run: | + PACKAGE="${{ inputs.package }}" + VERSION="${{ inputs.version }}" + + echo "πŸ“¦ Validating Rust crate: $PACKAGE" + echo "Version: $VERSION" + echo "" + + # Check if package exists in workspace + if ! cargo metadata --format-version 1 | jq -e ".packages[] | select(.name == \"$PACKAGE\")" > /dev/null; then + echo "❌ Package '$PACKAGE' not found in workspace" + echo "" + echo "Available packages:" + cargo metadata --format-version 1 | jq -r '.packages[].name' | sort + exit 1 + fi + + # Get package information + CARGO_VERSION=$(cargo metadata --format-version 1 | jq -r ".packages[] | select(.name == \"$PACKAGE\") | .version") + CARGO_PATH=$(cargo metadata --format-version 1 | jq -r ".packages[] | select(.name == \"$PACKAGE\") | .manifest_path") + + echo "Current Cargo.toml version: $CARGO_VERSION" + echo "Target version: $VERSION" + echo "Manifest path: $CARGO_PATH" + + # Check version consistency + if [ "$CARGO_VERSION" != "$VERSION" ]; then + echo "⚠️ Warning: Cargo.toml version ($CARGO_VERSION) doesn't match target version ($VERSION)" + echo "Make sure to update Cargo.toml before publishing" + fi + + # Show package dependencies + echo "" + echo "Package dependencies:" + cargo tree -p "$PACKAGE" --depth 1 | head -20 + shell: bash + + - name: Build package + run: | + PACKAGE="${{ inputs.package }}" + + echo "πŸ”¨ Building package: $PACKAGE" + cargo build -p "$PACKAGE" --release + + # Verify the build + if [ $? -eq 0 ]; then + echo "βœ… Package built successfully" + else + echo "❌ Build failed" + exit 1 + fi + shell: bash + + - name: Verify package contents + run: | + PACKAGE="${{ inputs.package }}" + + echo "πŸ“‹ Package contents verification:" + echo "" + + # List what would be included in the package + cargo package -p "$PACKAGE" --list | head -50 + + echo "" + echo "Package size estimate:" + cargo package -p "$PACKAGE" --list | wc -l + echo "files would be included" + shell: bash + + - name: Publish to crates.io + env: + CARGO_REGISTRY_TOKEN: ${{ env.CARGO_REGISTRY_TOKEN }} + run: | + PACKAGE="${{ inputs.package }}" + VERSION="${{ inputs.version }}" + + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo "πŸ” Dry run - would publish crate: $PACKAGE" + echo "" + + # Run cargo publish in dry-run mode + cargo publish --dry-run -p "$PACKAGE" + + echo "" + echo "Would publish:" + echo " Package: $PACKAGE" + echo " Version: $VERSION" + echo " Registry: crates.io" + echo "" + echo "After publishing, users could use:" + echo ' [dependencies]' + echo " $PACKAGE = \"$VERSION\"" + echo "" + echo "Or with cargo add:" + echo " cargo add $PACKAGE@$VERSION" + else + if [ -z "$CARGO_REGISTRY_TOKEN" ]; then + echo "❌ CARGO_REGISTRY_TOKEN is not set" + exit 1 + fi + + echo "πŸ“¦ Publishing crate to crates.io..." + echo "Package: $PACKAGE" + echo "Version: $VERSION" + echo "" + + # Publish the package + cargo publish -p "$PACKAGE" + + if [ $? -eq 0 ]; then + echo "" + echo "βœ… Successfully published to crates.io" + echo "" + echo "Package: $PACKAGE v$VERSION" + echo "Registry: https://crates.io/crates/$PACKAGE" + echo "" + echo "Users can now use:" + echo ' [dependencies]' + echo " $PACKAGE = \"$VERSION\"" + echo "" + echo "Or with cargo add:" + echo " cargo add $PACKAGE@$VERSION" + echo "" + echo "View on crates.io: https://crates.io/crates/$PACKAGE/$VERSION" + else + echo "❌ Publishing failed" + exit 1 + fi + fi + shell: bash diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml new file mode 100644 index 0000000000..e380b69f86 --- /dev/null +++ b/.github/actions/rust/pre-merge/action.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: rust-pre-merge +description: Rust pre-merge testing and linting github iggy actions + +inputs: + task: + description: "Task to run (check, fmt, clippy, sort, machete, doctest, test, compat)" + required: true + component: + description: "Component name (for context)" + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + show-stats: true + + - name: Install tools for specific tasks + run: | + case "${{ inputs.task }}" in + sort) + cargo install cargo-sort --locked + ;; + machete) + cargo install cargo-machete --locked + ;; + esac + shell: bash + + # Individual lint tasks for parallel execution + - name: Cargo check + if: inputs.task == 'check' + run: cargo check --all --all-features + shell: bash + + - name: Cargo fmt + if: inputs.task == 'fmt' + run: cargo fmt --all -- --check + shell: bash + + - name: Cargo clippy + if: inputs.task == 'clippy' + run: cargo clippy --all-targets --all-features -- -D warnings + shell: bash + + - name: Cargo sort + if: inputs.task == 'sort' + run: cargo sort --check --workspace + shell: bash + + - name: Cargo machete + if: inputs.task == 'machete' + run: cargo machete --with-metadata + shell: bash + + - name: Doc test + if: inputs.task == 'doctest' + run: | + cargo test --locked --doc + cargo doc --no-deps --all-features --quiet + shell: bash + + - name: Install dependencies for Rust tests + if: inputs.task == 'test' && runner.os == 'Linux' + run: | + sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils + rm -f $HOME/.local/share/keyrings/* + echo -n "test" | gnome-keyring-daemon --unlock + shell: bash + + - name: Build and test + if: inputs.task == 'test' + run: | + # Build all targets first + cargo build --locked --all-targets + + # Run tests with nextest for better parallelization and output + # nextest automatically uses all available cores and provides better test output + cargo nextest run --locked --no-fail-fast + shell: bash + + - name: Backwards compatibility check + if: inputs.task == 'compat' + run: | + scripts/check-backwards-compat.sh \ + --master-ref master \ + --pr-ref ${{ github.sha }} \ + --port 8090 --wait-secs 180 + shell: bash diff --git a/.github/actions/utils/docker-buildx/action.yml b/.github/actions/utils/docker-buildx/action.yml new file mode 100644 index 0000000000..ac25655852 --- /dev/null +++ b/.github/actions/utils/docker-buildx/action.yml @@ -0,0 +1,303 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: docker-buildx +description: Multi-arch Docker build and push +inputs: + task: + description: "Task to run (build/publish/docker)" + required: false + default: "build" + component: + description: "Component key from .github/config/publish.yml (e.g. rust-server)" + required: false + default: "" + version: + description: "Version tag (use 'edge' for master post-merge)" + required: false + default: "test" + context: + description: "Build context" + required: false + default: "." + dry_run: + description: "Dry run mode" + required: false + default: "false" + libc: + description: "Libc to use (glibc/musl)" + required: false + default: "musl" + +runs: + using: "composite" + steps: + - name: Resolve component from publish.yml + id: config + shell: bash + run: | + set -euxo pipefail + + comp="${{ inputs.component }}" + image="" + dockerfile="" + cfg_platforms="" + + if [ -n "$comp" ]; then + if ! command -v yq >/dev/null 2>&1; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + wget -qO /tmp/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /tmp/yq" | sha256sum -c - || exit 1 + chmod +x /tmp/yq + sudo mv /tmp/yq /usr/local/bin/yq + fi + + components_b64="$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" + + JSON_B64="$components_b64" COMP="$comp" node -e ' + const b64 = process.env.JSON_B64 || ""; + const comp = process.env.COMP || ""; + if (!b64) { console.error("Missing publish.yml components"); process.exit(1); } + const cfg = JSON.parse(Buffer.from(b64, "base64").toString("utf8")); + const e = cfg[comp]; + if (!e) { console.error(`Component not found in publish.yml: ${comp || ""}`); process.exit(1); } + const out = { + image: e.image || "", + dockerfile: e.dockerfile || "Dockerfile", + platforms: Array.isArray(e.platforms) ? e.platforms.join(",") : (e.platforms || "") + }; + process.stdout.write(JSON.stringify(out)); + ' > /tmp/_dockercfg.json + + image="$(jq -r .image /tmp/_dockercfg.json)" + dockerfile="$(jq -r .dockerfile /tmp/_dockercfg.json)" + cfg_platforms="$(jq -r .platforms /tmp/_dockercfg.json)" + fi + + should_push="false" + if [ "${{ inputs.task }}" = "publish" ] && [ "${{ inputs.dry_run }}" != "true" ]; then + should_push="true" + fi + + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "dockerfile=$dockerfile" >> "$GITHUB_OUTPUT" + echo "cfg_platforms=$cfg_platforms" >> "$GITHUB_OUTPUT" + echo "should_push=$should_push" >> "$GITHUB_OUTPUT" + + echo "🐳 Config:" + echo " component: ${comp:-}" + echo " image: ${image:-}" + echo " dockerfile:${dockerfile:-}" + echo " push: ${should_push}" + echo " version: ${{ inputs.version }}" + + if [ -n "$comp" ] && [ -z "$image" ]; then + echo "Component '${comp}' missing image mapping in publish.yml" >&2 + exit 1 + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + network=host + image=moby/buildkit:latest + + - name: Login to Docker Hub + if: steps.config.outputs.should_push == 'true' + uses: docker/login-action@v3 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.config.outputs.image }} + tags: | + type=raw,value=${{ inputs.version }} + type=raw,value=latest,enable=${{ steps.config.outputs.should_push == 'true' && inputs.version != 'test' && inputs.version != 'edge' }} + type=sha,enable=${{ inputs.version == 'test' }} + + - name: Determine platforms + id: platforms + shell: bash + run: | + p_cfg="${{ steps.config.outputs.cfg_platforms }}" + if [ -n "$p_cfg" ]; then + platforms="$p_cfg" + elif [ "${{ steps.config.outputs.should_push }}" = "true" ]; then + platforms="linux/amd64,linux/arm64" + else + platforms="linux/amd64" + fi + echo "platforms=$platforms" >> "$GITHUB_OUTPUT" + echo "πŸ–₯️ Platforms: $platforms" + + + - name: Compose cache config (no registry on dry-run) + id: cachecfg + shell: bash + run: | + set -euo pipefail + comp="${{ inputs.component }}" + os="${{ runner.os }}" + img="${{ steps.config.outputs.image }}" + if [ "${{ inputs.dry_run }}" = "true" ]; then + # dry-run/forks: avoid Docker Hub completely + CACHE_TO=$( + printf '%s\n' \ + "type=gha,scope=buildkit-${comp}-${os},mode=max" \ + "type=gha,scope=buildkit-${comp},mode=max" \ + "type=inline" + ) + CACHE_FROM=$( + printf '%s\n' \ + "type=gha,scope=buildkit-${comp}-${os}" \ + "type=gha,scope=buildkit-${comp}" + ) + else + # CI on upstream: use registry + gha + inline + CACHE_TO=$( + printf '%s\n' \ + "type=registry,ref=${img}:buildcache,mode=max" \ + "type=registry,ref=${img}:buildcache-${os},mode=max" \ + "type=inline" \ + "type=gha,scope=buildkit-${comp}-${os},mode=max" + ) + CACHE_FROM=$( + printf '%s\n' \ + "type=registry,ref=${img}:buildcache" \ + "type=registry,ref=${img}:buildcache-${os}" \ + "type=registry,ref=${img}:latest" \ + "type=gha,scope=buildkit-${comp}-${os}" \ + "type=gha,scope=buildkit-${comp}" + ) + fi + + { + echo 'CACHE_TO<> "$GITHUB_ENV" + + echo "Computed cache-to:" + printf '%s\n' "$CACHE_TO" + echo "Computed cache-from:" + printf '%s\n' "$CACHE_FROM" + + + - name: Determine Rust toolchain version (labels) + shell: bash + run: | + if [[ -f rust-toolchain.toml ]]; then + ver="$(sed -En 's/^[[:space:]]*channel[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' rust-toolchain.toml | head -1)" + : "${ver:=unknown}" + echo "RUST_VERSION=$ver" >> "$GITHUB_ENV" + echo "Using toolchain: $ver" + else + echo "RUST_VERSION=unknown" >> "$GITHUB_ENV" + echo "No rust-toolchain.toml found; labeling as unknown" + fi + + - name: Compose build args + id: bargs + shell: bash + run: | + set -euo pipefail + args="VERSION=${{ inputs.version }} + BUILD_DATE=${{ github.event.repository.updated_at }} + VCS_REF=${{ github.sha }} + RUST_VERSION=${{ env.RUST_VERSION }} + LIBC=${{ inputs.libc }} + IGGY_CI_BUILD=true" + { + echo "all<<__EOF__" + echo "$args" + echo "__EOF__" + } >> "$GITHUB_OUTPUT" + echo "Build args:" + printf '%s\n' "$args" + + - name: Determine build context + id: ctx + shell: bash + run: | + # For web-ui, the context should be the web directory + if [ "${{ inputs.component }}" = "web-ui" ]; then + context="web" + else + context="${{ inputs.context }}" + fi + echo "context=$context" >> "$GITHUB_OUTPUT" + echo "πŸ“ Build context: $context" + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: ${{ steps.ctx.outputs.context }} + file: ${{ steps.config.outputs.dockerfile }} + platforms: ${{ steps.platforms.outputs.platforms }} + push: ${{ steps.config.outputs.should_push }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: ${{ env.CACHE_FROM }} + cache-to: ${{ env.CACHE_TO }} + build-args: | + ${{ steps.bargs.outputs.all }} + + - name: Export image (if not pushing) + if: steps.config.outputs.should_push == 'false' && inputs.task != 'publish' + shell: bash + run: | + echo "Built: ${{ steps.config.outputs.image }}:${{ inputs.version }}" + echo "To save: docker save ${{ steps.config.outputs.image }}:${{ inputs.version }} -o image.tar" + + - name: Summary + if: always() + shell: bash + run: | + { + echo "## 🐳 Docker Build Summary" + echo "" + echo "| Property | Value |" + echo "|----------|-------|" + echo "| Image | \`${{ steps.config.outputs.image }}\` |" + echo "| Version | \`${{ inputs.version }}\` |" + echo "| Dockerfile | \`${{ steps.config.outputs.dockerfile }}\` |" + echo "| Platforms | \`${{ steps.platforms.outputs.platforms }}\` |" + echo "| Pushed | ${{ steps.config.outputs.should_push }} |" + if [ "${{ steps.config.outputs.should_push }}" = "true" ]; then + echo "| Digest | \`${{ steps.build.outputs.digest }}\` |" + echo "" + echo "### Pull" + echo '```bash' + echo "docker pull ${{ steps.config.outputs.image }}:${{ inputs.version }}" + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml new file mode 100644 index 0000000000..1ebe5350d2 --- /dev/null +++ b/.github/actions/utils/server-start/action.yml @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: server-start +description: Start Iggy server and wait for readiness +inputs: + mode: + description: "How to run the server: cargo|bin" + required: false + default: "cargo" + cargo-bin: + description: "Cargo binary name (when mode=cargo)" + required: false + default: "iggy-server" + cargo-profile: + description: "Cargo profile: release|debug" + required: false + default: "debug" + bin: + description: "Path to server binary (when mode=bin)" + required: false + default: "" + working-directory: + description: "Working directory for building/running" + required: false + default: "." + host: + description: "Bind host to test readiness against" + required: false + default: "127.0.0.1" + port: + description: "TCP port to check readiness" + required: false + default: "8090" + wait-timeout-seconds: + description: "Max seconds to wait until the server is ready" + required: false + default: "45" + log-file: + description: "Where to write server logs" + required: false + default: "" + pid-file: + description: "Where to write the PID" + required: false + default: "" + fail-if-busy: + description: "Fail if port is already in use" + required: false + default: "true" +outputs: + pid: + description: "PID of the background server process" + value: ${{ steps.out.outputs.pid }} + pid_file: + description: "Path to PID file" + value: ${{ steps.out.outputs.pid_file }} + log_file: + description: "Path to log file" + value: ${{ steps.out.outputs.log_file }} + address: + description: "Host:port used for checks" + value: ${{ steps.out.outputs.address }} +runs: + using: "composite" + steps: + - id: prep + shell: bash + run: | + set -euo pipefail + : "${RUNNER_TEMP:?}" + LOG_FILE="${{ inputs.log-file || '' }}" + PID_FILE="${{ inputs.pid-file || '' }}" + [ -n "$LOG_FILE" ] || LOG_FILE="$RUNNER_TEMP/iggy-server.log" + [ -n "$PID_FILE" ] || PID_FILE="$RUNNER_TEMP/iggy-server.pid" + echo "LOG_FILE=$LOG_FILE" >> "$GITHUB_ENV" + echo "PID_FILE=$PID_FILE" >> "$GITHUB_ENV" + + - id: resolve-bin + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + set -euo pipefail + MODE="${{ inputs.mode }}" + BIN_PATH="" + if [[ "$MODE" == "cargo" ]]; then + PROFILE="${{ inputs.cargo-profile }}" + NAME="${{ inputs.cargo-bin }}" + if [[ "$PROFILE" == "release" ]]; then + OUT="target/release/$NAME" + else + OUT="target/debug/$NAME" + fi + if [[ ! -x "$OUT" ]]; then + echo "Building $NAME with cargo ($PROFILE)…" + cargo build --locked --bin "$NAME" $([[ "$PROFILE" == "release" ]] && echo "--release") + fi + BIN_PATH="$OUT" + else + BIN_PATH="${{ inputs.bin }}" + [[ -x "$BIN_PATH" ]] || { echo "Binary not found or not executable: $BIN_PATH"; exit 1; } + fi + echo "bin=$BIN_PATH" >> "$GITHUB_OUTPUT" + + - id: busy + shell: bash + run: | + set -euo pipefail + HOST="${{ inputs.host }}" + PORT="${{ inputs.port }}" + # true if socket is already open + if command -v nc >/dev/null 2>&1; then + if nc -z "$HOST" "$PORT" 2>/dev/null; then busy=1; else busy=0; fi + else + if timeout 1 bash -lc ":/dev/null; then busy=1; else busy=0; fi + fi + echo "BUSY=$busy" >> "$GITHUB_ENV" + + - if: env.BUSY == '1' && inputs.fail-if-busy == 'true' + shell: bash + run: | + echo "Port ${{ inputs.host }}:${{ inputs.port }} is already in use." >&2 + exit 1 + + - name: Start server + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + set -euo pipefail + nohup "${{ steps.resolve-bin.outputs.bin }}" >"$LOG_FILE" 2>&1 & + echo $! > "$PID_FILE" + echo "Started server PID $(cat "$PID_FILE") β†’ logs: $LOG_FILE" + + - name: Wait for readiness + shell: bash + run: | + set -euo pipefail + HOST="${{ inputs.host }}" + PORT="${{ inputs.port }}" + DEADLINE=$(( $(date +%s) + ${{ inputs.wait-timeout-seconds }} )) + until (( $(date +%s) > DEADLINE )); do + if command -v nc >/dev/null 2>&1; then + nc -z "$HOST" "$PORT" 2>/dev/null && ready=1 || ready=0 + else + timeout 1 bash -lc ":/dev/null && ready=1 || ready=0 + fi + if [[ "$ready" == "1" ]]; then + echo "Server is ready on $HOST:$PORT" + break + fi + sleep 1 + done + if [[ "${ready:-0}" != "1" ]]; then + echo "Server did not become ready within ${{ inputs.wait-timeout-seconds }}s." >&2 + echo "---- last 100 lines of log ----" + tail -n 100 "$LOG_FILE" || true + kill "$(cat "$PID_FILE")" 2>/dev/null || true + exit 1 + fi + + - id: out + shell: bash + run: | + echo "pid=$(cat "$PID_FILE")" >> "$GITHUB_OUTPUT" + echo "pid_file=$PID_FILE" >> "$GITHUB_OUTPUT" + echo "log_file=$LOG_FILE" >> "$GITHUB_OUTPUT" + echo "address=${{ inputs.host }}:${{ inputs.port }}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/utils/server-stop/action.yml b/.github/actions/utils/server-stop/action.yml new file mode 100644 index 0000000000..3c694a1e9d --- /dev/null +++ b/.github/actions/utils/server-stop/action.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: server-stop +description: Stop Iggy server by PID or PID file +inputs: + pid: + description: "PID to stop (optional if pid-file is provided)" + required: false + default: "" + pid-file: + description: "Path to PID file" + required: false + default: "" + log-file: + description: "Path to log file (for final tail)" + required: false + default: "" + kill-timeout-seconds: + description: "Graceful timeout before SIGKILL" + required: false + default: "10" +runs: + using: "composite" + steps: + - shell: bash + run: | + set -euo pipefail + PID_INPUT="${{ inputs.pid }}" + PID_FILE="${{ inputs.pid-file }}" + LOG_FILE="${{ inputs.log-file }}" + if [[ -z "$PID_INPUT" && -n "$PID_FILE" && -f "$PID_FILE" ]]; then + PID_INPUT="$(cat "$PID_FILE" || true)" + fi + if [[ -z "$PID_INPUT" ]]; then + echo "No PID provided/found; nothing to stop." + exit 0 + fi + + if ps -p "$PID_INPUT" > /dev/null 2>&1; then + kill "$PID_INPUT" || true + for i in $(seq 1 ${{ inputs.kill-timeout-seconds }}); do + ps -p "$PID_INPUT" > /dev/null 2>&1 || exit 0 + sleep 1 + done + echo "Process still alive after grace; sending SIGKILL" + kill -9 "$PID_INPUT" || true + fi + + if [[ -n "$LOG_FILE" && -f "$LOG_FILE" ]]; then + echo "---- last 50 lines of server log ----" + tail -n 50 "$LOG_FILE" || true + fi diff --git a/.github/actions/utils/setup-rust-with-cache/action.yml b/.github/actions/utils/setup-rust-with-cache/action.yml new file mode 100644 index 0000000000..bf2df2831f --- /dev/null +++ b/.github/actions/utils/setup-rust-with-cache/action.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: this action sets up the Rust toolchain + swatinem/rust-cache for dependencies + sccache for compilation, +# it is a convenience wrapper, so that we can use it in all workflows. + +name: setup-rust-with-cache +description: Setup Rust toolchain and comprehensive caching (rust-cache for dependencies + sccache for compilation) +inputs: + enabled: + description: "Whether to enable caching" + required: false + default: "true" + show-stats: + description: "Whether to show sccache statistics at the end" + required: false + default: "false" + cache-targets: + description: "Whether to cache target directory (passed to rust-cache)" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Setup Rust toolchain + run: | + echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" + shell: bash + + - name: Setup Rust dependencies cache + if: inputs.enabled == 'true' + uses: Swatinem/rust-cache@v2 + with: + cache-targets: ${{ inputs.cache-targets }} + + - name: Setup sccache cache + if: inputs.enabled == 'true' + uses: actions/cache@v4 + with: + path: | + ~/.cache/sccache + ~/Library/Caches/sccache + ~/.local/share/sccache + key: sccache-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} + restore-keys: | + sccache-${{ runner.os }}-${{ github.job }}- + sccache-${{ runner.os }}- + + - name: Install sccache + if: inputs.enabled == 'true' + run: | + # Check if sccache is already installed + if ! command -v sccache &> /dev/null; then + echo "Installing sccache..." + SCCACHE_VERSION="v0.8.2" + SCCACHE_URL="https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" + + curl -L "$SCCACHE_URL" | tar xz + sudo mv sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache /usr/local/bin/ + rm -rf sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl + fi + shell: bash + continue-on-error: true + + - name: Install cargo-nextest + run: | + # Check if cargo-nextest is already installed + if ! command -v cargo-nextest &> /dev/null; then + echo "Installing cargo-nextest..." + # Use the install script for the fastest installation + curl -LsSf https://get.nexte.st/latest/linux | tar xzf - -C ${CARGO_HOME:-~/.cargo}/bin + echo "βœ… cargo-nextest installed successfully" + else + echo "βœ… cargo-nextest already installed" + fi + + # Verify installation + cargo nextest --version || true + shell: bash + continue-on-error: true + + - name: Configure Rust for sccache + if: inputs.enabled == 'true' + run: | + # Only use sccache if it was successfully installed + if command -v sccache &> /dev/null; then + # Configure sccache to use local disk cache + export SCCACHE_DIR="${HOME}/.cache/sccache" + mkdir -p "$SCCACHE_DIR" + + echo "SCCACHE_DIR=$SCCACHE_DIR" >> $GITHUB_ENV + echo "SCCACHE_CACHE_SIZE=2G" >> $GITHUB_ENV + echo "SCCACHE_ERROR_BEHAVIOR=warn" >> $GITHUB_ENV + echo "SCCACHE_IGNORE_SERVER_IO_ERROR=1" >> $GITHUB_ENV + + # Don't use GHA cache backend - use local disk cache with actions/cache + # This provides better control and persistence + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + + # Start sccache server + sccache --stop-server 2>/dev/null || true + sccache --start-server || true + + # Test if sccache is working + if sccache --show-stats >/dev/null 2>&1; then + echo "βœ… sccache configured with local cache at $SCCACHE_DIR" + + # Show initial stats + echo "Initial cache stats:" + sccache --show-stats || true + else + echo "⚠️ sccache installed but not functioning, continuing without cache" + fi + else + echo "⚠️ sccache not available, continuing without cache" + fi + shell: bash + + - name: Show sccache stats on completion + if: always() && inputs.enabled == 'true' && inputs.show-stats == 'true' + run: | + if command -v sccache &> /dev/null; then + echo "### Final sccache Statistics ###" + sccache --show-stats || true + fi + shell: bash diff --git a/.github/changed-files-config.json b/.github/changed-files-config.json deleted file mode 100644 index 87113c31bc..0000000000 --- a/.github/changed-files-config.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "rust": [ - "Dockerfile", - "Dockerfile.debug", - "core/(bench|cli|examples|integration|sdk|server|tools|connectors|ai).*\\.rs", - "bdd/rust/.*\\.rs", - "core/Cargo.toml", - "Cargo.toml", - "Cargo.lock", - ".github/workflows/ci-.*-rust.yml", - ".github/workflows/ci-check-pr.yml" - ], - "shell": [ - "Dockerfile", - "Dockerfile.debug", - ".github/scripts/.*\\.sh", - "scripts/.*\\.sh", - ".github/workflows/ci-.*-shell.yml", - ".github/workflows/ci-check-pr.yml" - ], - "rust-prod": [ - "Dockerfile", - "Dockerfile.debug", - "core/(bench|cli|examples|integration|sdk|server|tools|connectors|ai).*\\.rs", - "bdd/rust/.*\\.rs", - "core/Cargo.toml", - "Cargo.toml", - "Cargo.lock", - "Dockerfile.*", - ".github/workflows/ci-prod-rust.yml" - ], - "go-sdk": [ - "Dockerfile", - "Dockerfile.debug", - "foreign/go/.*\\.go", - "foreign/go/go.mod", - "foreign/go/go.sum", - "bdd/go/.*\\.go", - "bdd/go/go.mod", - "bdd/go/go.sum", - "examples/go/.*\\.go", - "examples/go/go.mod", - "examples/go/go.sum", - ".github/workflows/ci-check-go-sdk.yml" - ], - "java-sdk": [ - "Dockerfile", - "Dockerfile.debug", - "foreign/java/.*\\.java", - "foreign/java/.*\\.kts", - "foreign/java/.*\\.sh", - ".github/workflows/ci-check-java-sdk.yml" - ], - "python-sdk": [ - "Dockerfile", - "Dockerfile.debug", - "foreign/python/.*\\.py", - "foreign/python/.*\\.rs", - "bdd/python/.*\\.py", - "foreign/python/pyproject.toml", - "foreign/python/Cargo.toml", - "foreign/python/docker-compose.test.yml", - "foreign/python/Dockerfile.test", - "foreign/python/.*\\.toml", - "foreign/python/.*\\.ini", - "foreign/python/.*\\.md", - ".github/workflows/ci-check-python-sdk.yml" - ], - "node-sdk": [ - "Dockerfile", - "Dockerfile.debug", - "foreign/node/.*\\.ts", - "foreign/node/.*\\.js", - "foreign/node/.*\\.json", - ".github/workflows/ci-check-node-sdk.yml" - ], - "csharp-sdk": [ - "Dockerfile", - "Dockerfile.debug", - "foreign/csharp/.*\\.cs", - "foreign/csharp/.*\\.csproj", - ".github/workflows/ci-check-csharp-sdk.yml" - ], - "bdd": [ - "Dockerfile", - "Dockerfile.debug", - "bdd/.*\\.feature", - "bdd/.*\\.py", - "bdd/.*\\.rs", - "bdd/.*\\.go", - "bdd/go/go.mod", - "bdd/go/go.sum", - "bdd/.*Dockerfile", - "bdd/docker-compose.yml", - "scripts/run-bdd-tests.sh", - ".github/workflows/ci-test-bdd.yml", - "core/.*\\.rs", - "foreign/csharp/Iggy_SDK.Tests.BDD/.*\\.cs" - ] -} diff --git a/.github/config/components.yml b/.github/config/components.yml new file mode 100644 index 0000000000..de2d40b4f4 --- /dev/null +++ b/.github/config/components.yml @@ -0,0 +1,278 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +components: + # Workspace-level changes that affect all Rust components + rust-workspace: + paths: + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - ".cargo/**" + + # CI/CD infrastructure changes that require full regression + ci-infrastructure: + paths: + - ".github/**" + + # Core library components that others depend on + rust-sdk: + depends_on: + - "rust-workspace" # SDK is affected by workspace changes + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "core/sdk/**" + + rust-common: + depends_on: + - "rust-workspace" # Common is affected by workspace changes + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "core/common/**" + + rust-binary-protocol: + depends_on: + - "rust-workspace" # Protocol is affected by workspace changes + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "core/binary_protocol/**" + + rust-server: + depends_on: + - "rust-workspace" # Server is affected by workspace changes + - "ci-infrastructure" # CI changes trigger full regression + - "rust-common" + - "rust-binary-protocol" + paths: + - "core/server/**" + + # Main Rust workspace testing + rust: + depends_on: + - "rust-workspace" + - "rust-sdk" + - "rust-common" + - "rust-binary-protocol" + - "rust-server" + - "rust-tools" + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "Dockerfile*" + - "!foreign/**" # Exclude foreign SDKs + - "!web/**" # Exclude web UI + tasks: + - "check" + - "fmt" + - "clippy" + - "sort" + - "doctest" + - "machete" + - "test" + - "compat" + + # CLI component + rust-cli: + depends_on: + - "rust-sdk" + - "rust-binary-protocol" + paths: + - "core/cli/**" + + # Benchmark component + rust-bench: + depends_on: + - "rust-sdk" + paths: + - "core/bench/**" + + # Connectors runtime + rust-connectors: + depends_on: + - "rust-sdk" + paths: + - "core/connectors/**" + + # MCP AI component + rust-mcp: + depends_on: + - "rust-sdk" + paths: + - "core/ai/mcp/**" + + # Integration tests + rust-integration: + depends_on: + - "rust-sdk" + - "rust-common" + - "rust-binary-protocol" + - "rust-server" + paths: + - "core/integration/**" + + sdk-python: + depends_on: + - "rust-sdk" # Python SDK wraps the Rust SDK + - "rust-server" # For integration tests + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "foreign/python/**" + tasks: ["lint", "test", "build"] + + sdk-node: + depends_on: + - "rust-sdk" # Node SDK depends on core SDK + - "rust-server" # For integration tests + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "foreign/node/**" + tasks: ["lint", "test", "build", "e2e"] + + sdk-go: + depends_on: + - "rust-sdk" # Go SDK depends on core SDK + - "rust-server" # For integration tests + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "foreign/go/**" + - "bdd/go/**" + - "examples/go/**" + tasks: ["lint", "test", "build", "e2e"] + + sdk-java: + depends_on: + - "rust-sdk" # Java SDK depends on core SDK + - "rust-server" # For integration tests + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "foreign/java/**" + tasks: ["lint", "test", "build"] + + sdk-csharp: + depends_on: + - "rust-sdk" # C# SDK depends on core SDK + - "rust-server" # For integration tests + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "foreign/csharp/**" + tasks: ["lint", "test", "build"] + + # Individual BDD tests per SDK - only run when specific SDK changes + bdd-rust: + depends_on: + - "rust-sdk" + - "rust-server" + - "ci-infrastructure" + paths: + - "bdd/rust/**" + - "bdd/scenarios/**" + tasks: ["bdd-rust"] + + bdd-python: + depends_on: + - "rust-server" + - "rust-sdk" # All SDKs depend on core SDK changes + - "sdk-python" + - "ci-infrastructure" + paths: + - "bdd/python/**" + - "bdd/scenarios/**" + tasks: ["bdd-python"] + + bdd-go: + depends_on: + - "rust-server" + - "rust-sdk" # All SDKs depend on core SDK changes + - "sdk-go" + - "ci-infrastructure" + paths: + - "bdd/go/**" + - "bdd/scenarios/**" + tasks: ["bdd-go"] + + bdd-node: + depends_on: + - "rust-server" + - "rust-sdk" # All SDKs depend on core SDK changes + - "sdk-node" + - "ci-infrastructure" + paths: + - "bdd/node/**" + - "bdd/scenarios/**" + tasks: ["bdd-node"] + + bdd-csharp: + depends_on: + - "rust-server" + - "rust-sdk" # All SDKs depend on core SDK changes + - "sdk-csharp" + - "ci-infrastructure" + paths: + - "bdd/csharp/**" + - "bdd/scenarios/**" + tasks: ["bdd-csharp"] + + # Meta component for running all BDD tests (used for scripts/docker-compose changes) + bdd-suite: + depends_on: + - "bdd-rust" + - "bdd-python" + - "bdd-go" + - "bdd-node" + - "bdd-csharp" + paths: + - "bdd/docker-compose.yml" + - "bdd/Dockerfile" + - "scripts/run-bdd-tests.sh" + tasks: ["bdd-rust", "bdd-python", "bdd-go", "bdd-node", "bdd-csharp"] + + examples-suite: + depends_on: + - "rust-sdk" + - "rust-server" + - "sdk-go" + - "ci-infrastructure" # CI changes trigger full regression + paths: + - "examples/**" + - "scripts/run-rust-examples-from-readme.sh" + - "scripts/run-go-examples-from-readme.sh" + tasks: ["examples-rust", "examples-go"] + + web-ui: + paths: + - "web/**" + tasks: ["lint", "build"] + + rust-bench-dashboard: + paths: + - "core/bench/dashboard/**" + - "scripts/dashboard/**" + tasks: ["build"] + + # Core tools component + rust-tools: + depends_on: + - "rust-workspace" + paths: + - "core/tools/**" + + # CI/CD workflow monitoring (informational, no tasks) + ci-workflows: + paths: + - ".github/workflows/**/*.yml" + - ".github/actions/**/*.yml" + - ".github/ci/**/*.yml" + tasks: ["validate"] # Could run workflow validation diff --git a/.github/config/publish.yml b/.github/config/publish.yml new file mode 100644 index 0000000000..f536cb9e3e --- /dev/null +++ b/.github/config/publish.yml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +components: + + # ── Rust crates (crates.io) ──────────────────────────────────────────────── + rust-sdk: + tag_pattern: "^iggy-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: crates + package: iggy + version_file: "core/sdk/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-cli: + tag_pattern: "^cli-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: crates + package: iggy-cli + version_file: "core/cli/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-binary-protocol: + tag_pattern: "^binary-protocol-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: crates + package: iggy_binary_protocol + version_file: "core/binary_protocol/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-common: + tag_pattern: "^common-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: crates + package: iggy_common + version_file: "core/common/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + # ── DockerHub images ─────────────────────────────────────────────────────── + rust-server: + tag_pattern: "^server-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: dockerhub + image: apache/iggy + dockerfile: core/server/Dockerfile + platforms: ["linux/amd64", "linux/arm64"] + version_file: "core/server/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-mcp: + tag_pattern: "^mcp-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: dockerhub + image: apache/iggy-mcp + dockerfile: core/ai/mcp/Dockerfile + platforms: ["linux/amd64", "linux/arm64"] + version_file: "core/ai/mcp/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-bench-dashboard: + tag_pattern: "^bench-dashboard-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: dockerhub + image: apache/iggy-bench-dashboard + dockerfile: core/bench/dashboard/server/Dockerfile + platforms: ["linux/amd64"] + version_file: "core/bench/dashboard/server/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + rust-connectors: + tag_pattern: "^connectors-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: dockerhub + image: apache/iggy-connect + dockerfile: core/connectors/runtime/Dockerfile + platforms: ["linux/amd64", "linux/arm64"] + version_file: "core/connectors/runtime/Cargo.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + web-ui: + tag_pattern: "^web-ui-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: dockerhub + image: apache/iggy-web-ui + dockerfile: web/Dockerfile + platforms: ["linux/amd64"] + version_file: "web/package.json" + version_regex: '"version"\s*:\s*"([^"]+)"' + + # ── Other SDKs ───────────────────────────────────────────────────────────── + sdk-python: + tag_pattern: "^python-sdk-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: pypi + version_file: "foreign/python/pyproject.toml" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + sdk-node: + tag_pattern: "^node-sdk-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: npm + version_file: "foreign/node/package.json" + version_regex: '"version"\s*:\s*"([^"]+)"' + + sdk-java: + tag_pattern: "^java-sdk-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: maven + version_file: "foreign/java/java-sdk/build.gradle.kts" + version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' + + sdk-csharp: + tag_pattern: "^csharp-sdk-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: nuget + version_file: "foreign/csharp/Iggy_SDK/Iggy_SDK.csproj" + version_regex: '(?ms)<(?:PackageVersion|Version)>\s*([^<]+)\s*' + + # ── Go (tag only, no registry publish) ───────────────────────────────────── + sdk-go: + tag_pattern: "^foreign/go/v([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" + registry: none diff --git a/.github/scripts/analyze_changed_files.py b/.github/scripts/analyze_changed_files.py deleted file mode 100644 index ad2c8da582..0000000000 --- a/.github/scripts/analyze_changed_files.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import argparse -import json -import sys -import re -import os -import pprint - - -def load_config(config_file): - """Load the configuration from a JSON file.""" - try: - with open(config_file, 'r') as f: - return json.load(f) - except Exception as e: - print(f"Error loading config file: {e}") - sys.exit(1) - - -def categorize_files(files, config): - """Categorize files based on the configuration.""" - results = {} - matched_files = {} - - for category, patterns in config.items(): - print(f"Checking files for category: {category}") - results[category] = False - matched_files[category] = [] - for file in files: - print(f" Checking {file}") - for pattern in patterns: - if re.match(f"^{pattern}$", file) is not None: - print(f" Matched pattern: {pattern}") - results[category] = True - matched_files[category].append(file) - break - - return results, matched_files - - -def parse_arguments(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description= - 'Analyze changed files and categorize them based on patterns from a config file.\ - Outputs environment variables using GITHUB_ENV and GITHUB_OUTPUT for GitHub Actions to indicate\ - which categories have changed files. Environment variables are in the format:\ - CATEGORY_FILES_CHANGED=true and CATEGORY_FILES=file1.py\\nfile2.py\ - where CATEGORY is the name of the category in uppercase from the config file.' - ) - - parser.add_argument( - 'changed_files', - metavar='changed_files', - default=None, - help='String with list of file changed in the PR or commit') - - parser.add_argument( - 'config_file', - default=None, - help='Path to the JSON configuration file with file patterns and groups' - ) - - parser.add_argument('-v', - '--verbose', - action='store_true', - default=False, - help='Enable verbose output for debugging') - - return parser.parse_args() - - -def main(): - """Main function.""" - args = parse_arguments() - - changed_files = args.changed_files.splitlines() - config_file = args.config_file - verbose = args.verbose - config = load_config(config_file) - - if verbose: - print("Changed files:") - for file in changed_files: - print(f" {file}") - print("\nConfig file:", config_file) - pprint.pp(config) - print("") - - # Categorize files - results, matched_files = categorize_files(changed_files, config) - - # Set GitHub environment variables - github_env = os.environ.get('GITHUB_ENV') - github_output = os.environ.get('GITHUB_OUTPUT') - - # Output results - print("\nChanged Files Analysis results:") - - for category, has_changes in results.items(): - category_upper = category.upper() - print(f" {category_upper}_FILES_CHANGED={str(has_changes).lower()}") - - if github_env: - with open(github_env, 'a') as f: - f.write( - f"{category_upper}_FILES_CHANGED={str(has_changes).lower()}\n" - ) - - if has_changes: - f.write(f"{category_upper}_FILES</dev/null; then - echo "python3 is required but not installed, please install it" - exit 1 -fi - -if [ -z "${GITHUB_EVENT_NAME:-}" ]; then - # If the script is not running in a GitHub Actions environment (e.g., running locally), - # get the changed files based on the last commit - echo "The script is not running in a GitHub Actions environment" - CHANGED_FILES=$(git diff --name-only HEAD^) -else - # If the script is running in a GitHub Actions environment, check the event type - # Get the changed files based on the event type - if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then - # If the event type is push, get the changed files based on the last commit - echo "The script is running in a GitHub Actions environment for a push event" - CHANGED_FILES=$(git diff --name-only HEAD^) - else - # If the event type is not push (assuming pull request), get the changed files based - # on the base and head refs of the pull request. If the GITHUB_BASE_REF and GITHUB_HEAD_REF - # environment variables are not set, exit the script with an error message. - if [[ -z "${GITHUB_BASE_REF:-}" || -z "${GITHUB_HEAD_REF:-}" ]]; then - echo "The GITHUB_BASE_REF or GITHUB_HEAD_REF environment variable is not set" - exit 1 - fi - # Get the changed files based on the base and head refs of the pull request - echo "The script is running in a GitHub Actions environment for a pull request event" - echo "Base ref: origin/${GITHUB_BASE_REF}, Head ref: HEAD" - - # Fetch the base branch to ensure we have it - git fetch origin "${GITHUB_BASE_REF}" --depth=50 || true - - # Get the merge base to ensure we're comparing the right commits - MERGE_BASE=$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD) - echo "Merge base: ${MERGE_BASE}" - - # Get changed files from the merge base to HEAD - CHANGED_FILES=$(git diff --name-only "${MERGE_BASE}" HEAD) - - if [ -z "$CHANGED_FILES" ]; then - echo "No changed files detected. Trying alternative method..." - # Alternative: compare against the target branch directly - CHANGED_FILES=$(git diff --name-only "origin/${GITHUB_BASE_REF}...HEAD") - fi - - echo "Changed files detected:" - echo "$CHANGED_FILES" - fi -fi - -# Analyze the changed files -# shellcheck disable=SC2086 -python3 .github/scripts/analyze_changed_files.py ${VERBOSE_FLAG} "$CHANGED_FILES" .github/changed-files-config.json diff --git a/.github/workflows/_build_python_wheels.yml b/.github/workflows/_build_python_wheels.yml new file mode 100644 index 0000000000..40d83413e9 --- /dev/null +++ b/.github/workflows/_build_python_wheels.yml @@ -0,0 +1,282 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# .github/workflows/_build_python_wheels.yml +name: _build_python_wheels +on: + workflow_call: + inputs: + version: + type: string + required: false + default: "" + upload_artifacts: + type: boolean + required: false + default: true + use_latest_ci: + type: boolean + required: false + default: true + description: "Use latest CI configuration and scripts from master branch" + outputs: + artifact_name: + description: "Name of the uploaded artifact containing wheels" + value: ${{ jobs.collect.outputs.artifact_name }} + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + + - uses: actions/checkout@v4 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ 2>/dev/null || true + cp -r /tmp/latest-github/* .github/ + if [ -d scripts ]; then chmod +x scripts/*.sh 2>/dev/null || true; fi + echo "βœ… Latest CI and scripts applied" + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + working-directory: foreign/python + before-script-linux: | + # manylinux_2_28 uses yum, not dnf + if command -v dnf &> /dev/null; then + dnf install -y perl-IPC-Cmd + elif command -v yum &> /dev/null; then + yum install -y perl-IPC-Cmd + fi + python3 -m pip --version || python3 -m ensurepip + manylinux: "2_28" + args: --release --out dist --interpreter python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 + sccache: "true" + + - name: Upload wheels + if: inputs.upload_artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.target }} + path: foreign/python/dist + retention-days: 7 + + macos: + runs-on: ${{ matrix.target == 'x86_64' && 'macos-13' || 'macos-14' }} + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + + - uses: actions/checkout@v4 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ 2>/dev/null || true + cp -r /tmp/latest-github/* .github/ + if [ -d scripts ]; then chmod +x scripts/*.sh 2>/dev/null || true; fi + echo "βœ… Latest CI and scripts applied" + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + working-directory: foreign/python + args: --release --out dist --interpreter python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 + sccache: "true" + + - name: Upload wheels + if: inputs.upload_artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.target }} + path: foreign/python/dist + retention-days: 7 + + windows: + runs-on: windows-latest + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + shell: bash + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + + - uses: actions/checkout@v4 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + shell: bash + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ 2>/dev/null || true + cp -r /tmp/latest-github/* .github/ + if [ -d scripts ]; then chmod +x scripts/*.sh 2>/dev/null || true; fi + echo "βœ… Latest CI and scripts applied" + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: x64 + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: x86_64 + working-directory: foreign/python + args: --release --out dist --interpreter python3.9 python3.10 python3.11 python3.12 python3.13 + sccache: "true" + + - name: Upload wheels + if: inputs.upload_artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-windows-x64 + path: foreign/python/dist + retention-days: 7 + + sdist: + runs-on: ubuntu-latest + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + + - uses: actions/checkout@v4 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ 2>/dev/null || true + cp -r /tmp/latest-github/* .github/ + if [ -d scripts ]; then chmod +x scripts/*.sh 2>/dev/null || true; fi + echo "βœ… Latest CI and scripts applied" + + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + working-directory: foreign/python + args: --out dist + + - name: Upload sdist + if: inputs.upload_artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: foreign/python/dist + retention-days: 7 + + collect: + name: Collect all wheels + needs: [linux, macos, windows, sdist] + runs-on: ubuntu-latest + outputs: + artifact_name: ${{ steps.output.outputs.artifact_name }} + steps: + - name: Download all wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist + + - name: List wheels + run: | + echo "## πŸ“¦ Built Python Wheels" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Platform | Architecture | File |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------------|------|" >> $GITHUB_STEP_SUMMARY + + for wheel in dist/*.whl; do + filename=$(basename "$wheel") + if [[ "$filename" == *"linux"* ]]; then platform="Linux" + elif [[ "$filename" == *"macosx"* ]]; then platform="macOS" + elif [[ "$filename" == *"win"* ]]; then platform="Windows" + else platform="Universal"; fi + + if [[ "$filename" == *"x86_64"* ]] || [[ "$filename" == *"amd64"* ]]; then arch="x86_64" + elif [[ "$filename" == *"aarch64"* ]] || [[ "$filename" == *"arm64"* ]]; then arch="arm64" + elif [[ "$filename" == *"i686"* ]]; then arch="x86" + else arch="any"; fi + + echo "| $platform | $arch | \`$filename\` |" >> $GITHUB_STEP_SUMMARY + done + + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Total wheels built:** $(ls -1 dist/*.whl | wc -l)" >> $GITHUB_STEP_SUMMARY + + - name: Upload combined artifact + uses: actions/upload-artifact@v4 + with: + name: python-wheels-all + path: dist + retention-days: 30 + + - id: output + run: echo "artifact_name=python-wheels-all" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_common.yml b/.github/workflows/_common.yml new file mode 100644 index 0000000000..199dfcbe82 --- /dev/null +++ b/.github/workflows/_common.yml @@ -0,0 +1,478 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _common +on: + workflow_call: + inputs: + skip_pr_title: + type: boolean + required: false + default: false + description: "Skip PR title check (for push events)" + +permissions: + contents: read + pull-requests: read + +jobs: + rust-versions: + name: Check Rust versions sync + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - uses: actions/checkout@v4 + + - name: Check Rust versions are synchronized + run: | + # Use the sync-rust-version.sh script in check mode + if ! bash scripts/sync-rust-version.sh --check; then + echo "" + echo "❌ Rust versions are not synchronized!" + echo "" + echo "To fix this issue, run:" + echo " ./scripts/sync-rust-version.sh --fix" + echo "" + echo "This script will automatically update all Dockerfiles to match rust-toolchain.toml" + exit 1 + fi + + pr-title: + name: Check PR Title + if: github.event_name == 'pull_request' && !inputs.skip_pr_title + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - name: Validate PR Title + uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + types: | + feat + fix + docs + style + refactor + perf + test + build + ci + chore + revert + repo + deps + + license-headers: + name: Check license headers + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - uses: actions/checkout@v4 + + - name: Check Apache license headers + run: | + echo "πŸ” Checking license headers..." + + # Pull the addlicense image + docker pull ghcr.io/google/addlicense:latest + + # Run the check + if docker run --rm -v ${{ github.workspace }}:/src -w /src \ + ghcr.io/google/addlicense:latest \ + -check -f ASF_LICENSE.txt . > missing_files.txt 2>&1; then + echo "βœ… All files have proper license headers" + else + file_count=$(wc -l < missing_files.txt) + echo "❌ Found $file_count files missing license headers:" + echo "" + cat missing_files.txt | sed 's/^/ β€’ /' + echo "" + echo "πŸ’‘ Run 'addlicense -f ASF_LICENSE.txt .' to fix automatically" + + # Add to summary + echo "## ❌ License Headers Missing" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The following files are missing Apache license headers:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat missing_files.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo 'Please call `just licenses-fix` to fix automatically.' >> $GITHUB_STEP_SUMMARY + + exit 1 + fi + + license-list: + name: Check licenses list + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: ./.github/actions/utils/setup-rust-with-cache + with: + enabled: "false" # Don't need cache for just checking licenses + + - run: scripts/licenses-list.sh --check + + markdown: + name: Markdown lint + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '23' + + - name: Install markdownlint-cli + run: npm install -g markdownlint-cli + + - name: Run markdownlint + run: | + echo "πŸ” Checking markdown files..." + + # Create config if it doesn't exist + if [ ! -f ".markdownlint.yml" ]; then + cat > .markdownlint.yml << 'EOF' + # Markdown lint configuration + default: true + MD013: + line_length: 120 + tables: false + MD033: + allowed_elements: [details, summary, img] + MD041: false # First line in file should be a top level heading + EOF + fi + + # Run the linter + if markdownlint '**/*.md' --ignore-path .gitignore; then + echo "βœ… All markdown files are properly formatted" + else + echo "❌ Markdown linting failed" + echo "πŸ’‘ Run 'markdownlint **/*.md --fix' to auto-fix issues" + exit 1 + fi + + shellcheck: + name: Shell scripts lint + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install shellcheck + run: | + sudo apt-get update --yes && sudo apt-get install --yes shellcheck + + - name: Check shell scripts + run: | + echo "πŸ” Checking shell scripts..." + + # Find all shell scripts excluding certain directories + if find . -type f -name "*.sh" \ + -not -path "./target/*" \ + -not -path "./node_modules/*" \ + -not -path "./.git/*" \ + -not -path "./foreign/node/node_modules/*" \ + -not -path "./foreign/python/.venv/*" \ + -exec shellcheck -S warning {} +; then + echo "βœ… All shell scripts passed shellcheck" + else + echo "❌ Shellcheck found issues in shell scripts" + echo "πŸ’‘ Fix the issues reported above" + exit 1 + fi + + trailing-whitespace: + name: Check trailing whitespace + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full history to get diff + + - name: Check for trailing whitespace in changed files + run: | + echo "πŸ” Checking for trailing whitespace in changed files..." + + # Get list of changed files in PR + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch --no-tags --depth=1 origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} || true + BASE_SHA="${{ github.event.pull_request.base.sha }}" + CHANGED_FILES=$(git diff --name-only --diff-filter=ACM "$BASE_SHA"...HEAD || true) + else + CHANGED_FILES=$(git diff --name-only --diff-filter=ACM HEAD~1) + fi + + if [ -z "$CHANGED_FILES" ]; then + echo "No files changed to check" + exit 0 + fi + + echo "Files to check:" + echo "$CHANGED_FILES" | sed 's/^/ β€’ /' + echo "" + + # Check each changed file for trailing whitespace + FILES_WITH_TRAILING="" + for file in $CHANGED_FILES; do + # Skip if file doesn't exist (might be deleted) + if [ ! -f "$file" ]; then + continue + fi + + # Skip binary files + if file "$file" | grep -qE "binary|data|executable|compressed"; then + continue + fi + + # Check for trailing whitespace + if grep -q '[[:space:]]$' "$file" 2>/dev/null; then + FILES_WITH_TRAILING="$FILES_WITH_TRAILING $file" + fi + done + + if [ -z "$FILES_WITH_TRAILING" ]; then + echo "βœ… No trailing whitespace found in changed files" + else + echo "❌ Found trailing whitespace in the following changed files:" + echo "" + for file in $FILES_WITH_TRAILING; do + echo " β€’ $file" + # Show lines with trailing whitespace (limit to first 5 occurrences per file) + grep -n '[[:space:]]$' "$file" | head -5 | while IFS=: read -r line_num content; do + # Show the line with visible whitespace markers + visible_content=$(echo "$content" | sed 's/ /Β·/g; s/\t/β†’/g') + echo " Line $line_num: '${visible_content}'" + done + TOTAL_LINES=$(grep -c '[[:space:]]$' "$file") + if [ "$TOTAL_LINES" -gt 5 ]; then + echo " ... and $((TOTAL_LINES - 5)) more lines" + fi + echo "" + done + + echo "πŸ’‘ To fix trailing whitespace in these files:" + echo " β€’ VSCode: Enable 'files.trimTrailingWhitespace' setting" + echo " β€’ Fix specific file: sed -i 's/[[:space:]]*$//' " + echo " β€’ Fix all changed files:" + echo " for f in$FILES_WITH_TRAILING; do sed -i 's/[[:space:]]*$//' \$f; done" + exit 1 + fi + + trailing-newline: + name: Check trailing newline + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full history to get diff + + - name: Check for trailing newline in changed text files + run: | + echo "πŸ” Checking for trailing newline in changed text files..." + + # Get list of changed files in PR + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch --no-tags --depth=1 origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} || true + BASE_SHA="${{ github.event.pull_request.base.sha }}" + CHANGED_FILES=$(git diff --name-only --diff-filter=ACM "$BASE_SHA"...HEAD || true) + else + CHANGED_FILES=$(git diff --name-only --diff-filter=ACM HEAD~1) + fi + + if [ -z "$CHANGED_FILES" ]; then + echo "No files changed to check" + exit 0 + fi + + echo "Files to check:" + echo "$CHANGED_FILES" | sed 's/^/ β€’ /' + echo "" + + # Check each changed file for missing trailing newline + FILES_WITHOUT_NEWLINE="" + for file in $CHANGED_FILES; do + # Skip if file doesn't exist (might be deleted) + if [ ! -f "$file" ]; then + continue + fi + + # Skip binary files + if file "$file" | grep -qE "binary|data|executable|compressed"; then + continue + fi + + # Skip empty files + if [ ! -s "$file" ]; then + continue + fi + + # Check if file ends with a newline + # Use tail to get last byte and od to check if it's a newline (0x0a) + if [ -n "$(tail -c 1 "$file" | od -An -tx1 | grep -v '0a')" ]; then + FILES_WITHOUT_NEWLINE="$FILES_WITHOUT_NEWLINE $file" + fi + done + + if [ -z "$FILES_WITHOUT_NEWLINE" ]; then + echo "βœ… All changed text files have trailing newlines" + else + echo "❌ Found text files without trailing newline:" + echo "" + for file in $FILES_WITHOUT_NEWLINE; do + echo " β€’ $file" + # Show last few characters of the file for context + echo -n " Last characters: '" + tail -c 20 "$file" | tr '\n' '↡' | sed 's/\t/β†’/g' + echo "'" + echo "" + done + + echo "πŸ’‘ To add trailing newlines to these files:" + echo " β€’ VSCode: Enable 'files.insertFinalNewline' setting" + echo " β€’ Fix specific file: echo >> " + echo " β€’ Fix all files:" + echo " for f in$FILES_WITHOUT_NEWLINE; do [ -n \"\$(tail -c 1 \"\$f\")\" ] && echo >> \"\$f\"; done" + exit 1 + fi + + summary: + name: Common checks summary + needs: [rust-versions, pr-title, license-headers, license-list, markdown, shellcheck, trailing-whitespace, trailing-newline] + if: always() + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + steps: + - name: Summary + run: | + echo "## πŸ“‹ Common Checks Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Check | Status | Description |" >> $GITHUB_STEP_SUMMARY + echo "|-------|--------|-------------|" >> $GITHUB_STEP_SUMMARY + + # PR-specific checks + if [ "${{ github.event_name }}" = "pull_request" ]; then + PR_TITLE="${{ needs.pr-title.result }}" + + # Add emoji based on status + if [ "$PR_TITLE" = "success" ]; then + echo "| βœ… PR Title | success | Follows conventional format |" >> $GITHUB_STEP_SUMMARY + elif [ "$PR_TITLE" = "failure" ]; then + echo "| ❌ PR Title | failure | Must follow conventional format |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ PR Title | $PR_TITLE | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + else + echo "| ⏭️ PR Title | skipped | Not a pull request |" >> $GITHUB_STEP_SUMMARY + fi + + # Always-run checks + RUST_VERSIONS="${{ needs.rust-versions.result }}" + LICENSE_HEADERS="${{ needs.license-headers.result }}" + LICENSE_LIST="${{ needs.license-list.result }}" + MARKDOWN="${{ needs.markdown.result }}" + + if [ "$RUST_VERSIONS" = "success" ]; then + echo "| βœ… Rust Versions | success | All Rust versions synchronized |" >> $GITHUB_STEP_SUMMARY + elif [ "$RUST_VERSIONS" = "failure" ]; then + echo "| ❌ Rust Versions | failure | Rust versions mismatch in Dockerfiles |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ Rust Versions | $RUST_VERSIONS | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$LICENSE_HEADERS" = "success" ]; then + echo "| βœ… License Headers | success | All files have Apache headers |" >> $GITHUB_STEP_SUMMARY + elif [ "$LICENSE_HEADERS" = "failure" ]; then + echo "| ❌ License Headers | failure | Missing Apache license headers |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ License Headers | $LICENSE_HEADERS | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$LICENSE_LIST" = "success" ]; then + echo "| βœ… License List | success | Dependencies licenses validated |" >> $GITHUB_STEP_SUMMARY + elif [ "$LICENSE_LIST" = "failure" ]; then + echo "| ❌ License List | failure | License list needs update |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ License List | $LICENSE_LIST | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$MARKDOWN" = "success" ]; then + echo "| βœ… Markdown Lint | success | All markdown files are valid |" >> $GITHUB_STEP_SUMMARY + elif [ "$MARKDOWN" = "failure" ]; then + echo "| ❌ Markdown Lint | failure | Markdown formatting issues found |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ Markdown Lint | $MARKDOWN | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + SHELLCHECK="${{ needs.shellcheck.result }}" + if [ "$SHELLCHECK" = "success" ]; then + echo "| βœ… Shellcheck | success | All shell scripts are valid |" >> $GITHUB_STEP_SUMMARY + elif [ "$SHELLCHECK" = "failure" ]; then + echo "| ❌ Shellcheck | failure | Shell script issues found |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ Shellcheck | $SHELLCHECK | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + TRAILING="${{ needs.trailing-whitespace.result }}" + if [ "$TRAILING" = "success" ]; then + echo "| βœ… Trailing Whitespace | success | No trailing whitespace found |" >> $GITHUB_STEP_SUMMARY + elif [ "$TRAILING" = "failure" ]; then + echo "| ❌ Trailing Whitespace | failure | Trailing whitespace detected |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ Trailing Whitespace | $TRAILING | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + TRAILING_NL="${{ needs.trailing-newline.result }}" + if [ "$TRAILING_NL" = "success" ]; then + echo "| βœ… Trailing Newline | success | All text files have trailing newlines |" >> $GITHUB_STEP_SUMMARY + elif [ "$TRAILING_NL" = "failure" ]; then + echo "| ❌ Trailing Newline | failure | Missing trailing newlines detected |" >> $GITHUB_STEP_SUMMARY + else + echo "| ⏭️ Trailing Newline | $TRAILING_NL | Check skipped |" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + + # Overall status + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "### ❌ Some checks failed" >> $GITHUB_STEP_SUMMARY + echo "Please review the failed checks above and fix the issues." >> $GITHUB_STEP_SUMMARY + elif [[ "${{ contains(needs.*.result, 'skipped') }}" == "true" ]]; then + echo "### ⚠️ Some checks were skipped" >> $GITHUB_STEP_SUMMARY + else + echo "### βœ… All checks passed!" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/_detect.yml b/.github/workflows/_detect.yml new file mode 100644 index 0000000000..0d11c5e3db --- /dev/null +++ b/.github/workflows/_detect.yml @@ -0,0 +1,382 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# .github/workflows/_detect.yml +name: _detect +on: + workflow_call: + outputs: + rust_matrix: + description: "Matrix for Rust components" + value: ${{ jobs.detect.outputs.rust_matrix }} + python_matrix: + description: "Matrix for Python SDK" + value: ${{ jobs.detect.outputs.python_matrix }} + node_matrix: + description: "Matrix for Node SDK" + value: ${{ jobs.detect.outputs.node_matrix }} + go_matrix: + description: "Matrix for Go SDK" + value: ${{ jobs.detect.outputs.go_matrix }} + java_matrix: + description: "Matrix for Java SDK" + value: ${{ jobs.detect.outputs.java_matrix }} + csharp_matrix: + description: "Matrix for C# SDK" + value: ${{ jobs.detect.outputs.csharp_matrix }} + bdd_matrix: + description: "Matrix for BDD tests" + value: ${{ jobs.detect.outputs.bdd_matrix }} + examples_matrix: + description: "Matrix for examples tests" + value: ${{ jobs.detect.outputs.examples_matrix }} + other_matrix: + description: "Matrix for other components" + value: ${{ jobs.detect.outputs.other_matrix }} + +jobs: + detect: + runs-on: ubuntu-latest + env: + IGGY_CI_BUILD: true + outputs: + rust_matrix: ${{ steps.mk.outputs.rust_matrix }} + python_matrix: ${{ steps.mk.outputs.python_matrix }} + node_matrix: ${{ steps.mk.outputs.node_matrix }} + go_matrix: ${{ steps.mk.outputs.go_matrix }} + java_matrix: ${{ steps.mk.outputs.java_matrix }} + csharp_matrix: ${{ steps.mk.outputs.csharp_matrix }} + bdd_matrix: ${{ steps.mk.outputs.bdd_matrix }} + examples_matrix: ${{ steps.mk.outputs.examples_matrix }} + other_matrix: ${{ steps.mk.outputs.other_matrix }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get changed files (PR or push) + id: changed + uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c + with: + format: json + + - name: Load components config + id: config + run: | + if ! command -v yq &> /dev/null; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + wget -qO /tmp/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /tmp/yq" | sha256sum -c - || exit 1 + chmod +x /tmp/yq + sudo mv /tmp/yq /usr/local/bin/yq + fi + echo "components=$(yq -o=json -I=0 '.' .github/config/components.yml | jq -c)" >> $GITHUB_OUTPUT + + - name: Build matrices + id: mk + uses: actions/github-script@v7 + with: + script: | + const componentsJson = `${{ steps.config.outputs.components }}`; + const changedFilesJson = `${{ steps.changed.outputs.all || '[]' }}`; + + let componentsCfg = { components: {} }; + try { componentsCfg = JSON.parse(componentsJson); } catch {} + const components = componentsCfg.components || {}; + + let changedFiles = []; + try { changedFiles = JSON.parse(changedFilesJson); } catch {} + const files = changedFiles.map(p => p.replace(/\\/g, '/')); + + const escapeRe = s => s.replace(/([.+^${}()|[\]\\])/g, '\\$1'); + + const globToRegex = (pattern) => { + // Handle exclusion patterns + const isExclusion = pattern.startsWith('!'); + const cleanPattern = isExclusion ? pattern.slice(1) : pattern; + + let s = escapeRe(cleanPattern); + + // First, replace ** patterns with placeholders to protect them + s = s.replace(/\*\*\/+/g, '___DOUBLESTAR_SLASH___'); // '**/' -> placeholder + s = s.replace(/\/\*\*$/g, '___SLASH_DOUBLESTAR___'); // '/**' at end -> placeholder + s = s.replace(/\*\*/g, '___DOUBLESTAR___'); // remaining '**' -> placeholder + + // Now handle single * and ? (they won't match our placeholders) + s = s.replace(/\*/g, '[^/]*'); // '*' -> match any except / + s = s.replace(/\?/g, '[^/]'); // '?' -> match single char except / + + // Finally, replace placeholders with actual regex patterns + s = s.replace(/___DOUBLESTAR_SLASH___/g, '(?:.*/)?'); // '**/' -> any subpath (optional) + s = s.replace(/___SLASH_DOUBLESTAR___/g, '(?:/.*)?'); // '/**' at end -> rest of path + s = s.replace(/___DOUBLESTAR___/g, '.*'); // remaining '**' -> match anything + + return { regex: new RegExp(`^${s}$`), exclude: isExclusion }; + }; + + const compiled = new Map(); + const toRx = (p) => { + if (!compiled.has(p)) compiled.set(p, globToRegex(p)); + return compiled.get(p).regex; + }; + const test = (file, patterns) => { + const inc = patterns.filter(p => !p.startsWith('!')).map(toRx); + const exc = patterns.filter(p => p.startsWith('!')).map(p => toRx(p.slice(1))); + const included = inc.some(rx => rx.test(file)); + const excluded = exc.some(rx => rx.test(file)); + return included && !excluded; + }; + + // Build dependency graph and resolve affected components + const affectedComponents = new Set(); + const componentTriggers = new Map(); // Track what triggered each component + const dependencyGraph = new Map(); + + // First pass: build dependency graph + for (const [name, cfg] of Object.entries(components)) { + if (cfg.depends_on) { + dependencyGraph.set(name, cfg.depends_on); + } + } + + // Second pass: check which components are directly affected by file changes + console.log('=== Change Detection ==='); + console.log(`Analyzing ${files.length} changed files...`); + console.log(''); + + for (const [name, cfg] of Object.entries(components)) { + const pats = (cfg.paths || []); + const matchedFiles = files.filter(f => test(f, pats)); + if (matchedFiles.length > 0) { + affectedComponents.add(name); + componentTriggers.set(name, { + reason: 'direct', + files: matchedFiles, + dependency: null + }); + + // Log direct matches + console.log(`βœ“ ${name} directly affected by:`); + if (matchedFiles.length <= 5) { + matchedFiles.forEach(f => console.log(` - ${f}`)); + } else { + matchedFiles.slice(0, 3).forEach(f => console.log(` - ${f}`)); + console.log(` ... and ${matchedFiles.length - 3} more files`); + } + } + } + + // Third pass: resolve transitive dependencies + console.log(''); + console.log('=== Dependency Resolution ==='); + + const resolveDependent = (componentName, depth = 0) => { + for (const [dependent, dependencies] of dependencyGraph.entries()) { + if (dependencies.includes(componentName) && !affectedComponents.has(dependent)) { + affectedComponents.add(dependent); + + // Track why this component was added + if (!componentTriggers.has(dependent)) { + componentTriggers.set(dependent, { + reason: 'dependency', + files: [], + dependency: componentName + }); + console.log(`${' '.repeat(depth)}β†’ ${dependent} (depends on ${componentName})`); + } + + resolveDependent(dependent, depth + 1); // Recursively add dependents + } + } + }; + + // Apply dependency resolution + const initialAffected = [...affectedComponents]; + for (const comp of initialAffected) { + resolveDependent(comp, 1); + } + + // Summary output + console.log(''); + console.log('=== Summary ==='); + console.log(`Initially affected: ${initialAffected.length} components`); + console.log(`After dependencies: ${affectedComponents.size} components`); + + if (files.length <= 10) { + console.log(''); + console.log('Changed files:'); + files.forEach(f => console.log(` - ${f}`)); + } else { + console.log(`Total files changed: ${files.length}`); + } + + const groups = { rust:[], python:[], node:[], go:[], java:[], csharp:[], bdd:[], examples:[], other:[] }; + + // Process affected components and generate tasks + console.log(''); + console.log('=== Task Generation ==='); + + const skippedComponents = []; + const taskedComponents = []; + + for (const name of affectedComponents) { + const cfg = components[name]; + const trigger = componentTriggers.get(name); + + if (!cfg || !cfg.tasks || cfg.tasks.length === 0) { + skippedComponents.push(name); + continue; + } + + const entries = cfg.tasks.map(task => ({ component: name, task })); + taskedComponents.push({ name, tasks: cfg.tasks, trigger }); + + if (name === 'rust') groups.rust.push(...entries); + else if (name === 'sdk-python') groups.python.push(...entries); + else if (name === 'sdk-node') groups.node.push(...entries); + else if (name === 'sdk-go') groups.go.push(...entries); + else if (name === 'sdk-java') groups.java.push(...entries); + else if (name === 'sdk-csharp') groups.csharp.push(...entries); + else if (name.startsWith('bdd-')) { + // Individual BDD tests should run separately with proper Docker setup + groups.bdd.push(...entries); + } + else if (name === 'examples-suite') { + // Examples should run separately + groups.examples.push(...entries); + } + else if (name === 'shell-scripts') groups.other.push(...entries); + else if (name === 'ci-workflows') groups.other.push(...entries); + else groups.other.push(...entries); + } + + // Log components with tasks + if (taskedComponents.length > 0) { + console.log('Components with tasks to run:'); + taskedComponents.forEach(({ name, tasks, trigger }) => { + const reason = trigger.reason === 'direct' + ? `directly triggered by ${trigger.files.length} file(s)` + : `dependency of ${trigger.dependency}`; + console.log(` βœ“ ${name}: ${tasks.length} task(s) - ${reason}`); + }); + } + + // Log skipped components (no tasks defined) + if (skippedComponents.length > 0) { + console.log(''); + console.log('Components triggered but skipped (no tasks):'); + skippedComponents.forEach(name => { + const trigger = componentTriggers.get(name); + const reason = trigger.reason === 'direct' + ? `directly triggered by ${trigger.files.length} file(s)` + : `dependency of ${trigger.dependency}`; + console.log(` β—‹ ${name} - ${reason}`); + }); + } + + // On master push, run everything + if (context.eventName === 'push' && context.ref === 'refs/heads/master') { + // Clear existing groups to avoid duplicates - we'll run everything anyway + groups.rust = []; + groups.python = []; + groups.node = []; + groups.go = []; + groups.java = []; + groups.csharp = []; + groups.bdd = []; + groups.examples = []; + groups.other = []; + + for (const [name, cfg] of Object.entries(components)) { + if (!cfg.tasks || cfg.tasks.length === 0) continue; + const entries = cfg.tasks.map(task => ({ component: name, task })); + if (name === 'rust') groups.rust.push(...entries); + else if (name === 'sdk-python') groups.python.push(...entries); + else if (name === 'sdk-node') groups.node.push(...entries); + else if (name === 'sdk-go') groups.go.push(...entries); + else if (name === 'sdk-java') groups.java.push(...entries); + else if (name === 'sdk-csharp') groups.csharp.push(...entries); + else if (name.startsWith('bdd-')) groups.bdd.push(...entries); + else if (name === 'examples-suite') groups.examples.push(...entries); + else groups.other.push(...entries); + } + } + + // Deduplicate entries in each group (in case of any edge cases) + const dedupeGroup = (group) => { + const seen = new Set(); + return group.filter(item => { + const key = `${item.component}-${item.task}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + }; + + // Apply deduplication to all groups + Object.keys(groups).forEach(key => { + groups[key] = dedupeGroup(groups[key]); + }); + + const matrix = a => a.length ? { include: a } : { include: [{ component: 'noop', task: 'noop' }] }; + + // Final summary of what will run + console.log(''); + console.log('=== Test Jobs to Run ==='); + + const jobSummary = [ + { name: 'Rust', tasks: groups.rust }, + { name: 'Python SDK', tasks: groups.python }, + { name: 'Node SDK', tasks: groups.node }, + { name: 'Go SDK', tasks: groups.go }, + { name: 'Java SDK', tasks: groups.java }, + { name: 'C# SDK', tasks: groups.csharp }, + { name: 'BDD Tests', tasks: groups.bdd }, + { name: 'Examples', tasks: groups.examples }, + { name: 'Other', tasks: groups.other } + ]; + + jobSummary.forEach(({ name, tasks }) => { + if (tasks.length > 0) { + const uniqueTasks = [...new Set(tasks.map(t => t.task))]; + console.log(` βœ“ ${name}: ${uniqueTasks.join(', ')}`); + } else { + console.log(` βœ— ${name}: SKIPPED (no changes detected)`); + } + }); + + // Overall stats + const totalTasks = Object.values(groups).reduce((sum, g) => sum + g.length, 0); + console.log(''); + console.log(`Total: ${affectedComponents.size} components affected, ${totalTasks} tasks to run`); + + // Use environment files instead of deprecated setOutput + const setOutput = (name, value) => { + const output = `${name}=${value}`; + require('fs').appendFileSync(process.env.GITHUB_OUTPUT, `${output}\n`); + }; + + setOutput('rust_matrix', JSON.stringify(matrix(groups.rust))); + setOutput('python_matrix', JSON.stringify(matrix(groups.python))); + setOutput('node_matrix', JSON.stringify(matrix(groups.node))); + setOutput('go_matrix', JSON.stringify(matrix(groups.go))); + setOutput('java_matrix', JSON.stringify(matrix(groups.java))); + setOutput('csharp_matrix', JSON.stringify(matrix(groups.csharp))); + setOutput('bdd_matrix', JSON.stringify(matrix(groups.bdd))); + setOutput('examples_matrix', JSON.stringify(matrix(groups.examples))); + setOutput('other_matrix', JSON.stringify(matrix(groups.other))); diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 0000000000..d496ef7b0e --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _test +on: + workflow_call: + inputs: + component: + type: string + required: true + description: "Component to test" + task: + type: string + required: true + description: "Task to run" + +permissions: + contents: read + security-events: write + +jobs: + run: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Skip noop + if: inputs.component == 'noop' + run: echo "No changes detected, skipping tests" + + # Rust + - name: Run Rust task + if: startsWith(inputs.component, 'rust') + uses: ./.github/actions/rust/pre-merge + with: + task: ${{ inputs.task }} + component: ${{ inputs.component }} + + # Python SDK + - name: Set up Docker Buildx for Python + if: inputs.component == 'sdk-python' && inputs.task == 'test' + uses: docker/setup-buildx-action@v3 + + - name: Run Python SDK task + if: inputs.component == 'sdk-python' + uses: ./.github/actions/python-maturin/pre-merge + with: + task: ${{ inputs.task }} + + # Node SDK + - name: Run Node SDK task + if: inputs.component == 'sdk-node' + uses: ./.github/actions/node-npm/pre-merge + with: + task: ${{ inputs.task }} + + # Go SDK + - name: Run Go SDK task + if: inputs.component == 'sdk-go' + uses: ./.github/actions/go/pre-merge + with: + task: ${{ inputs.task }} + + # Java SDK + - name: Run Java SDK task + if: inputs.component == 'sdk-java' + uses: ./.github/actions/java-gradle/pre-merge + with: + task: ${{ inputs.task }} + + # C# SDK + - name: Run C# SDK task + if: inputs.component == 'sdk-csharp' + uses: ./.github/actions/csharp-dotnet/pre-merge + with: + task: ${{ inputs.task }} + + # Web UI + - name: Run Web UI task + if: inputs.component == 'web-ui' + run: | + cd web + npm ci + if [ "${{ inputs.task }}" = "lint" ]; then + # TODO(hubcio): make this blocking once Web UI lints are fixed + npm run lint || true + elif [ "${{ inputs.task }}" = "build" ]; then + npm run build + fi + + # CI workflow validation + - name: Validate CI workflows + if: inputs.component == 'ci-workflows' && inputs.task == 'validate' + run: | + echo "Validating GitHub Actions workflows..." + # Basic YAML validation + for workflow in .github/workflows/*.yml; do + echo "Checking $workflow" + python -c "import yaml; yaml.safe_load(open('$workflow'))" || exit 1 + done + echo "All workflows are valid YAML" + + # Upload reports + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/** + target/llvm-cov/** + coverage.lcov + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml new file mode 100644 index 0000000000..6a1c25e690 --- /dev/null +++ b/.github/workflows/_test_bdd.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _test_bdd +on: + workflow_call: + inputs: + component: + type: string + required: true + description: "Component to test (e.g., 'bdd-rust', 'bdd-python', etc.)" + task: + type: string + required: true + description: "Task to run (e.g., 'bdd-rust', 'bdd-python', etc.)" + +permissions: + contents: read + security-events: write + +jobs: + run: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Skip noop + if: inputs.component == 'noop' + run: echo "No changes detected, skipping tests" + + - name: Setup Rust with cache for BDD + if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + + - name: Build server for BDD tests + if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') + run: | + echo "Building server binary and CLI for BDD tests with sccache..." + cargo build --locked --bin iggy-server --bin iggy + + echo "Server binary built at: target/debug/iggy-server" + ls -lh target/debug/iggy-server + + echo "CLI binary built at: target/debug/iggy" + ls -lh target/debug/iggy + + # Verify the binary exists and is executable + if [ ! -f "target/debug/iggy-server" ]; then + echo "ERROR: Server binary not found at target/debug/iggy-server" + exit 1 + fi + + if [ ! -f "target/debug/iggy" ]; then + echo "ERROR: CLI binary not found at target/debug/iggy" + exit 1 + fi + + - name: Run BDD tests + if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') + run: | + # Extract SDK name from task (format: bdd-) + SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/bdd-//') + + echo "Running BDD tests for SDK: $SDK_NAME" + echo "Current directory: $(pwd)" + echo "Server binary location: $(ls -lh target/debug/iggy-server)" + echo "CLI binary location: $(ls -lh target/debug/iggy)" + + # Export path to the pre-built server and cli binaries (relative to repo root) + export IGGY_SERVER_PATH="target/debug/iggy-server" + export IGGY_CLI_PATH="target/debug/iggy" + ./scripts/run-bdd-tests.sh "$SDK_NAME" + + - name: Clean up Docker resources (BDD) + if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') + run: | + ./scripts/run-bdd-tests.sh clean || true + docker system prune -f || true + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/** + target/llvm-cov/** + coverage.lcov + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml new file mode 100644 index 0000000000..5f23aa9998 --- /dev/null +++ b/.github/workflows/_test_examples.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: _test_examples +on: + workflow_call: + inputs: + component: + type: string + required: true + description: "Component to test (should be 'examples-suite')" + task: + type: string + required: true + description: "Task to run (e.g., 'examples-rust', 'examples-go')" + +permissions: + contents: read + security-events: write + +jobs: + run: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + IGGY_CI_BUILD: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Skip noop + if: inputs.component == 'noop' + run: echo "No changes detected, skipping tests" + + - name: Setup Rust with cache for examples + if: inputs.component == 'examples-suite' + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false # Only cache registry and git deps, not target dir (sccache handles that) + + - name: Build common binaries for all examples + if: inputs.component == 'examples-suite' + run: | + echo "Building common binaries for all examples tests..." + echo "Current directory: $(pwd)" + + # Build server (needed by both Rust and Go examples) + echo "Building iggy-server..." + cargo build --locked --bin iggy-server + + # For Rust examples, also build CLI and example binaries + if [[ "${{ inputs.task }}" == "examples-rust" ]]; then + echo "Building additional binaries for Rust examples..." + cargo build --locked --bin iggy --examples + fi + + # Verify server binary was built (needed by all examples) + echo "Verifying server binary:" + if [ -f "target/debug/iggy-server" ]; then + echo "βœ… Server binary found at target/debug/iggy-server" + ls -lh target/debug/iggy-server + else + echo "❌ Server binary NOT found at target/debug/iggy-server" + echo "Checking target directory structure:" + find target -name "iggy-server" -type f 2>/dev/null || true + exit 1 + fi + + # For Rust examples, verify CLI was built + if [[ "${{ inputs.task }}" == "examples-rust" ]]; then + if [ -f "target/debug/iggy" ]; then + echo "βœ… CLI binary found at target/debug/iggy" + ls -lh target/debug/iggy + else + echo "❌ CLI binary NOT found at target/debug/iggy" + exit 1 + fi + fi + + - name: Run Rust examples + if: inputs.component == 'examples-suite' && inputs.task == 'examples-rust' + run: | + echo "Running Rust examples tests..." + # Run the examples script which will use the prebuilt binaries + ./scripts/run-rust-examples-from-readme.sh + + - name: Run Go examples + if: inputs.component == 'examples-suite' && inputs.task == 'examples-go' + run: | + echo "Running Go examples tests..." + # Run the examples script which will use the prebuilt server binary + ./scripts/run-go-examples-from-readme.sh + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.component }}-${{ inputs.task }}-reports-${{ github.run_id }}-${{ github.run_attempt }} + path: | + reports/** + target/llvm-cov/** + coverage.lcov + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/ci-check-common.yml b/.github/workflows/ci-check-common.yml deleted file mode 100644 index 711bd186bf..0000000000 --- a/.github/workflows/ci-check-common.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# ------------------------------------------------------------- -# -# CI Check Common Workflow -# -# This workflow runs common code quality checks that are not specific -# to any programming language. Checks include: -# - PR title validation using conventional commits -# - license header verification -# - license list validation -# - markdown linting using markdownlint -# -name: ci-check-common - -on: - workflow_call: - inputs: - commits-from: - description: 'Lower end of the commit range to check' - required: true - default: HEAD~1 - type: string - commits-to: - description: 'Upper end of the new commit range to check' - required: true - default: HEAD - type: string - -jobs: - check-pr-title: - name: Check PR Title - runs-on: ubuntu-latest - steps: - - name: Validate PR Title - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - # Configure types as needed (optional, defaults to common ones) - types: | - feat - fix - docs - style - refactor - perf - test - build - ci - chore - revert - repo - - ci-check-licenses: - name: Check license headers - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Pull addlicense Docker image - run: docker pull ghcr.io/google/addlicense:latest - - - name: Check license headers - run: | - echo "πŸ” Checking license headers..." - - if docker run --rm -v ${{ github.workspace }}:/src -w /src ghcr.io/google/addlicense:latest -check -f ASF_LICENSE.txt . > missing_files.txt 2>&1; then - echo "βœ… All files have proper license headers" - else - file_count=$(wc -l < missing_files.txt) - echo "❌ Found $file_count files missing license headers:" - echo "" - cat missing_files.txt | sed 's/^/ β€’ /' - echo "" - echo "πŸ’‘ Run 'addlicense -f ASF_LICENSE.txt .' to fix automatically" - exit 1 - fi - - ci-check-licenses-list: - name: Check licenses list - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - run: scripts/licenses-list.sh --check - - markdownlint: - name: markdownlint - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - name: Install markdownlint-cli - run: | - npm install -g markdownlint-cli - - name: Run markdownlint - run: | - markdownlint '**/*.md' --ignore-path .gitignore diff --git a/.github/workflows/ci-check-csharp-sdk.yml b/.github/workflows/ci-check-csharp-sdk.yml deleted file mode 100644 index 5cb8571162..0000000000 --- a/.github/workflows/ci-check-csharp-sdk.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: ci-check-csharp-sdk - -on: - workflow_dispatch: - workflow_call: - -jobs: - build: - name: csharp build - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Setup .NET 8 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '8.0.x' - - name: Restore dependencies - working-directory: foreign/csharp - run: dotnet restore - - name: Build - working-directory: foreign/csharp - run: dotnet build --no-restore - - test: - name: Unit test - needs: build - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Setup .NET 8 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '8.0.x' - - name: Restore dependencies - working-directory: foreign/csharp - run: dotnet restore - - name: Build - working-directory: foreign/csharp - run: dotnet build --no-restore - - name: Test - working-directory: foreign/csharp - run: | - dotnet test Iggy_SDK_Tests --no-build --verbosity normal - dotnet test Iggy_SDK.Tests.Integration --no-build --verbosity normal diff --git a/.github/workflows/ci-check-go-sdk.yml b/.github/workflows/ci-check-go-sdk.yml deleted file mode 100644 index 08b2abd5f4..0000000000 --- a/.github/workflows/ci-check-go-sdk.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -name: ci-check-go-sdk - -on: - workflow_dispatch: - workflow_call: - -jobs: - build-and-test: - name: Go Build And Test - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.23.0' - - - name: Tidy Check - working-directory: foreign/go - run: | - go mod tidy - git diff --exit-code - - - name: Build Go Modules - working-directory: foreign/go - run: go build ./... - - - name: Unit Test - working-directory: foreign/go - run: go test -v ./... - - - name: Run and Verify CLI Commands from Go SDK Examples README - run: ./scripts/run-go-examples-from-readme.sh - - lint: - strategy: - matrix: - module: [ foreign/go, bdd/go, examples/go ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.23.0' - - name: golangci-lint - uses: golangci/golangci-lint-action@v8 - with: - version: v2.1 - working-directory: ${{ matrix.module }} \ No newline at end of file diff --git a/.github/workflows/ci-check-java-sdk.yml b/.github/workflows/ci-check-java-sdk.yml deleted file mode 100644 index a10498f42a..0000000000 --- a/.github/workflows/ci-check-java-sdk.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check Java Workflow -# -# This workflow runs checks for Java code. -# Checks include: -# - Java Gradle build -# - TODO: Add checkstyle verification, will be done after #1753 -# - TODO: Add findbugs -# - TODO: License header check -# - TODO: Add code coverage -# - TODO: Add shell check for scripts under foreign/java -# -# This workflow can be triggered manually or by other workflows. -# -name: ci-check-java-sdk - -on: - workflow_dispatch: - workflow_call: - -jobs: - build: - name: Java build - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Java - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - - # Configure Gradle for optimal use in GitHub Actions, including caching of downloaded dependencies. - # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - - name: Setup Gradle - uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 - - - name: Build - run: foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest - - test: - name: Unit test - needs: build - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Java - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - - # Configure Gradle for optimal use in GitHub Actions, including caching of downloaded dependencies. - # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - - name: Setup Gradle - uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 - - - name: Run tests - run: foreign/java/dev-support/checks/build.sh test diff --git a/.github/workflows/ci-check-node-sdk.yml b/.github/workflows/ci-check-node-sdk.yml deleted file mode 100644 index bb4ecfbd75..0000000000 --- a/.github/workflows/ci-check-node-sdk.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check Node Workflow -# -# This workflow runs checks for nodejs sdk code. -# Checks include: -# - npm install dependencies -# - commit-lint -# - typescript build -# - eslint -# - unit test -# - TODO: e2e test -# -# This workflow can be triggered manually or by other workflows. -# -name: ci-check-node-sdk - -on: - workflow_dispatch: - workflow_call: - -jobs: - install: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - cache-dependency-path: foreign/node/package-lock.json - node-version: 22 - cache: "npm" - - - uses: actions/cache@v4 - env: - cache-name: cache-node-modules - with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - - - name: install dependencies - run: cd foreign/node/ && npm ci --ignore-scripts # no husky hooks when in ci - - # - name: Validate current commit (last commit) message with commitlint - # if: github.event_name == 'push' - # run: npx commitlint --last --verbose - - # - name: Validate PR commits messages with commitlint - # if: github.event_name == 'pull_request' - # run: npx commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose - - - name: lint typescript code - run: cd foreign/node/ && npm run lint - - - name: build typescript - run: cd foreign/node/ && npm run build - - - name: run unit test - run: cd foreign/node/ && npm run test:unit - - ### Integration tests (e2e) - - # - name: Setup iggy - # uses: iggy-rs/setup-iggy@v1 - - # - name: Perform integration tests - # run: npm run test:e2e diff --git a/.github/workflows/ci-check-pr.yml b/.github/workflows/ci-check-pr.yml deleted file mode 100644 index 543c1329f2..0000000000 --- a/.github/workflows/ci-check-pr.yml +++ /dev/null @@ -1,197 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check PR Workflow -# -# This workflow validates pull requests to the master branch by detecting changed files -# and running appropriate checks and tests. -# -# Flow: -# 1. pr-file-changes: Detects which file types were modified (mandatory) -# 2. ci-check-common: Validates commit message (mandatory) -# 3. Conditional jobs based on file changes: -# - For Rust changes: ci-check-rust β†’ ci-test-rust β†’ ci-test-rust-optional & ci-compatibility-rust -# - For shell changes: ci-check-shell -# 4. finalize-pr: Determines final PR status based on all job results (mandatory) -# -# Dependencies: -# - ci-check-rust depends on pr-file-changes (outputs.trigger-rust) -# - ci-test-rust and ci-compatibility-rust depend on ci-check-rust success -# - ci-check-shell depends on pr-file-changes (outputs.trigger-shell) -# - finalize-pr depends on all other jobs -# -# The workflow fails if any mandatory job fails. -# Workflow can be triggered manually or on pull request events. - -name: ci-check-pr - -on: - workflow_dispatch: - pull_request: - branches: - - master - types: [ opened, synchronize, reopened ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - pr-file-changes: - name: pr-file-changes - runs-on: ubuntu-latest - outputs: - trigger-rust: ${{ steps.changed-files.outputs.RUST_FILES_CHANGED }} - trigger-shell: ${{ steps.changed-files.outputs.SHELL_FILES_CHANGED }} - trigger-java-sdk: ${{ steps.changed-files.outputs.JAVA-SDK_FILES_CHANGED }} - trigger-python-sdk: ${{ steps.changed-files.outputs.PYTHON-SDK_FILES_CHANGED }} - trigger-go-sdk: ${{ steps.changed-files.outputs.GO-SDK_FILES_CHANGED }} - trigger-node-sdk: ${{ steps.changed-files.outputs.NODE-SDK_FILES_CHANGED }} - trigger-csharp-sdk: ${{ steps.changed-files.outputs.CSHARP-SDK_FILES_CHANGED }} - trigger-bdd: ${{ steps.changed-files.outputs.BDD_FILES_CHANGED }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install python3 - run: | - sudo apt-get update - sudo apt-get install -y python3 - python3 --version - - name: Detect file changes - id: changed-files - run: | - .github/scripts/detect-changed-files.sh master - - name: List all changed files - run: | - if [ "${{ steps.changed-files.outputs.RUST_FILES_CHANGED }}" == "true" ]; then - echo "One or more rust file(s) has changed." - echo "List all the files that have changed: ${{ steps.changed-files.outputs.RUST_FILES }}" - fi - if [ "${{ steps.changed-files.outputs.SHELL_FILES_CHANGED }}" == "true" ]; then - echo "One or more shell file(s) has changed." - echo "List all the files that have changed: ${{ steps.changed-files.outputs.SHELL_FILES }}" - fi - - ci-check-common: - name: ci-check-common - uses: ./.github/workflows/ci-check-common.yml - with: - commits-from: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || 'HEAD~1' }} - commits-to: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || 'HEAD' }} - - ci-check-rust: - name: ci-check-rust - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-rust == 'true' }} - uses: ./.github/workflows/ci-check-rust.yml - - ci-test-rust: - name: ci-test-rust - needs: ci-check-rust - if: ${{ needs.ci-check-rust.result == 'success' }} - uses: ./.github/workflows/ci-test-rust.yml - - ci-test-rust-optional: - name: ci-test-rust-optional - needs: ci-check-rust - if: ${{ needs.ci-check-rust.result == 'success' }} - uses: ./.github/workflows/ci-test-rust-optional.yml - - ci-compatibility-rust: - name: ci-compatibility-rust - needs: ci-check-rust - if: ${{ needs.ci-check-rust.result == 'success' }} - uses: ./.github/workflows/ci-compatibility-rust.yml - with: - pr_body: ${{ github.event.pull_request.body }} - - ci-check-shell: - name: ci-check-shell - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-shell == 'true' }} - uses: ./.github/workflows/ci-check-shell.yml - - ci-check-java-sdk: - name: ci-check-java-sdk - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-java-sdk == 'true' }} - uses: ./.github/workflows/ci-check-java-sdk.yml - - ci-check-python-sdk: - name: ci-check-python-sdk - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-python-sdk == 'true' }} - uses: ./.github/workflows/ci-check-python-sdk.yml - - ci-check-go-sdk: - name: ci-check-go-sdk - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-go-sdk == 'true' }} - uses: ./.github/workflows/ci-check-go-sdk.yml - ci-check-node-sdk: - name: ci-check-node-sdk - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-node-sdk == 'true' }} - uses: ./.github/workflows/ci-check-node-sdk.yml - - ci-check-csharp-sdk: - name: ci-check-csharp-sdk - needs: pr-file-changes - if: ${{ needs.pr-file-changes.outputs.trigger-csharp-sdk == 'true' }} - uses: ./.github/workflows/ci-check-csharp-sdk.yml - - ci-test-bdd: - name: ci-test-bdd - needs: pr-file-changes - if: ${{ - needs.pr-file-changes.outputs.trigger-rust == 'true' || - needs.pr-file-changes.outputs.trigger-java-sdk == 'true' || - needs.pr-file-changes.outputs.trigger-python-sdk == 'true' || - needs.pr-file-changes.outputs.trigger-go-sdk == 'true' || - needs.pr-file-changes.outputs.trigger-node-sdk == 'true' || - needs.pr-file-changes.outputs.trigger-csharp-sdk == 'true' || - needs.pr-file-changes.outputs.trigger-bdd == 'true' }} - uses: ./.github/workflows/ci-test-bdd.yml - - finalize_pr: - runs-on: ubuntu-latest - needs: - - pr-file-changes - - ci-check-common - - ci-check-rust - - ci-test-rust - - ci-compatibility-rust - - ci-check-shell - - ci-check-java-sdk - - ci-check-python-sdk - - ci-check-go-sdk - - ci-check-node-sdk - - ci-check-csharp-sdk - - ci-test-bdd - if: always() - steps: - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Some tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 diff --git a/.github/workflows/ci-check-python-sdk.yml b/.github/workflows/ci-check-python-sdk.yml deleted file mode 100644 index 4f7287ce25..0000000000 --- a/.github/workflows/ci-check-python-sdk.yml +++ /dev/null @@ -1,165 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check Python SDK Workflow -# -# This workflow is integrated from the maturin-generated python.yml -# and adapted to work within the monorepo structure. -# -name: ci-check-python-sdk - -on: - workflow_dispatch: - workflow_call: - push: - tags: - - "python-v*" - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-latest - strategy: - matrix: - target: [x86_64, aarch64] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - . -> target - foreign/python -> target - cache-on-failure: true - key: python-${{ matrix.target }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - working-directory: foreign/python - before-script-linux: "dnf install -y perl-IPC-Cmd && (python3 -m pip --version || python3 -m ensurepip)" - manylinux: "2_28" - args: --release --out dist --interpreter python3.8 python3.9 python3.10 python3.11 python3.12 python3.13 python3.14 - sccache: "true" - - name: Set up Docker Buildx - if: matrix.target == 'x86_64' - uses: docker/setup-buildx-action@v3 - - name: Run tests - if: matrix.target == 'x86_64' - working-directory: foreign/python - run: | - # Run tests using Docker Compose with caching - DOCKER_BUILDKIT=1 docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from python-tests - - # Clean up - docker compose -f docker-compose.test.yml down -v - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-linux-${{ matrix.target }} - path: foreign/python/dist - - # windows: - # runs-on: windows-latest - # strategy: - # matrix: - # target: [x64] - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-python@v5 - # with: - # python-version: "3.10" - # architecture: ${{ matrix.target }} - # - name: Set up NASM # Action disallowed - # uses: ilammy/setup-nasm@v1 - # - name: Build wheels - # uses: PyO3/maturin-action@v1 - # with: - # target: ${{ matrix.target }} - # working-directory: foreign/python - # args: --release --out dist --find-interpreter - # sccache: "true" - # - name: Upload wheels - # uses: actions/upload-artifact@v4 - # with: - # name: wheels-windows-${{ matrix.target }} - # path: foreign/python/dist - - macos: - runs-on: macos-latest - strategy: - matrix: - target: [x86_64, aarch64] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - working-directory: foreign/python - args: --release --out dist --find-interpreter - sccache: "true" - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-macos-${{ matrix.target }} - path: foreign/python/dist - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - working-directory: foreign/python - args: --out dist - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: wheels-sdist - path: foreign/python/dist - - release: - name: Release - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/python-v') - needs: [linux, macos, sdist] - steps: - - uses: actions/download-artifact@v4 - with: - pattern: wheels-* - merge-multiple: true - path: dist - - name: Publish to PyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing dist/* diff --git a/.github/workflows/ci-check-rust.yml b/.github/workflows/ci-check-rust.yml deleted file mode 100644 index b8e45eb678..0000000000 --- a/.github/workflows/ci-check-rust.yml +++ /dev/null @@ -1,164 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check Rust Workflow -# -# This workflow runs checks for Rust code using cargo commands. -# Checks include: -# - static analysis using `cargo check` -# - code formatting using `cargo fmt` -# - linting using `cargo clippy` -# - sorted dependencies check using `cargo sort` -# - documentation tests using `cargo test --doc` -# - documentation generation using `cargo doc` -# - unused dependencies check using `cargo machete` -# -# This workflow can be triggered manually or by other workflows. -# -name: ci-check-rust - -on: - workflow_dispatch: - workflow_call: - -env: - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - -jobs: - check-rust-version-sync: - name: check Rust version sync - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Check Rust versions are synchronized - run: | - # rust-toolchain.toml -> 1.89(.0) -> 1.89 - TOOLCHAIN_VERSION=$(sed -En 's/^[[:space:]]*channel[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' rust-toolchain.toml) - to_minor() { echo "$1" | sed -E 's/^([0-9]+)\.([0-9]+).*/\1.\2/'; } - TOOLCHAIN_MINOR=$(to_minor "$TOOLCHAIN_VERSION") - - # Extract X[.Y[.Z]] from Dockerfiles (ignores suffixes like -slim-bookworm), then -> X.Y - BDD_VERSION=$(sed -En 's/^FROM[[:space:]]+rust:([0-9.]+).*/\1/p' bdd/rust/Dockerfile | head -1) - BENCH_VERSION=$(sed -En 's/^FROM[[:space:]]+rust:([0-9.]+).*/\1/p' core/bench/dashboard/server/Dockerfile | head -1) - BDD_MINOR=$(to_minor "$BDD_VERSION") - BENCH_MINOR=$(to_minor "$BENCH_VERSION") - - # Fail if mismatched - if [ "$TOOLCHAIN_MINOR" != "$BDD_MINOR" ]; then - echo "ERROR: bdd/rust/Dockerfile uses $BDD_VERSION (-> $BDD_MINOR) but rust-toolchain.toml specifies $TOOLCHAIN_VERSION (-> $TOOLCHAIN_MINOR)" >&2 - exit 1 - fi - - if [ "$TOOLCHAIN_MINOR" != "$BENCH_MINOR" ]; then - echo "ERROR: core/bench/dashboard/server/Dockerfile uses $BENCH_VERSION (-> $BENCH_MINOR) but rust-toolchain.toml specifies $TOOLCHAIN_VERSION (-> $TOOLCHAIN_MINOR)" >&2 - exit 1 - fi - - echo "βœ“ All Rust versions are synchronized at $TOOLCHAIN_VERSION" - - check: - name: cargo check - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Run cargo check - run: | - cargo check - - fmt: - name: cargo fmt - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Run cargo fmt - run: | - cargo fmt --all -- --check - - clippy: - name: cargo clippy - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Run cargo clippy - run: | - cargo clippy --all-targets --all-features -- -D warnings - - sort: - name: cargo sort - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Install cargo-sort - uses: taiki-e/install-action@v2 - with: - tool: cargo-sort - - name: Run cargo sort - run: | - cargo sort --check --workspace - - doctest: - name: cargo test docs - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Run cargo test (doc) - run: | - cargo test --doc - - name: Run cargo doc - run: | - cargo doc --no-deps --all-features --quiet - - unused_dependencies: - name: cargo machete - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Install cargo-machete - uses: taiki-e/install-action@v2 - with: - tool: cargo-machete - - name: Run cargo machete - run: | - cargo machete --with-metadata diff --git a/.github/workflows/ci-check-shell.yml b/.github/workflows/ci-check-shell.yml deleted file mode 100644 index f09d549d52..0000000000 --- a/.github/workflows/ci-check-shell.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Check Shell Workflow -# -# This workflow runs shellcheck on shell scripts in the repository. -# This workflow can be triggered manually or by other workflows. -# -name: ci-check-shell -on: - workflow_dispatch: - workflow_call: - -jobs: - shellcheck: - name: shellcheck - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Install shellcheck on Linux - run: | - sudo apt-get update --yes && sudo apt-get install --yes shellcheck - - name: Check shell scripts - run: find scripts -type f -name "*.sh" -exec shellcheck {} + diff --git a/.github/workflows/ci-compatibility-rust.yml b/.github/workflows/ci-compatibility-rust.yml deleted file mode 100644 index 6a1d4c96d6..0000000000 --- a/.github/workflows/ci-compatibility-rust.yml +++ /dev/null @@ -1,188 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Compatibility Rust Workflow -# -# This workflow runs compatibility tests for Rust code on the master branch for pull requests. -# This workflow is triggered by the ci-check-pr workflow. It checks if BREAKING_CHANGE in the PR body -# and commit messages and skips the compatibility tests if found. -# -name: backwards_compatibility - -on: - workflow_call: - inputs: - pr_body: - description: 'Pull Request body' - required: true - type: string - -jobs: - check_commit_message: - runs-on: ubuntu-latest - outputs: - should_skip: ${{ steps.check_skip.outputs.skip == 'true' }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - - name: Print the event payload - run: cat "$GITHUB_EVENT_PATH" - - - name: Check if BREAKING_CHANGE is present in any commits body in the PR - id: check_commits - run: | - BREAKING_CHANGE_FOUND=false - COMMIT_RANGE=origin/master...HEAD - - for COMMIT in $(git log --format=%H $COMMIT_RANGE); do - COMMIT_MSG=$(git log --format=%B -n 1 $COMMIT) - COMMIT_MSG_SUBJECT=$(echo "$COMMIT_MSG" | head -n 1) - COMMIT_MSG_BODY=$(echo "$COMMIT_MSG" | tail -n +3) - echo "Commit $COMMIT, subject: $COMMIT_MSG_SUBJECT, body: $COMMIT_MSG_BODY" - if echo "$COMMIT_MSG_BODY" | grep -q "BREAKING_CHANGE"; then - BREAKING_CHANGE_FOUND=true - break - fi - done - - if $BREAKING_CHANGE_FOUND; then - echo "skip=true" >> $GITHUB_OUTPUT - echo "'BREAKING_CHANGE' found in commit message, setting skip=true" - else - echo "skip=false" >> $GITHUB_OUTPUT - echo "'BREAKING_CHANGE' not found in commit message, setting skip=false" - fi - - - name: Check if BREAKING_CHANGE is present in pull request body - id: check_pr_body - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ENV_PR_BODY: ${{ inputs.pr_body }} - run: | - PR_BODY="$ENV_PR_BODY" - echo "Pull Request body: $PR_BODY" - if [[ "$PR_BODY" == *"BREAKING_CHANGE"* ]]; then - echo "skip=true" >> $GITHUB_OUTPUT - echo "'BREAKING_CHANGE' found in pull request body, setting skip=true" - else - echo "skip=false" >> $GITHUB_OUTPUT - echo "'BREAKING_CHANGE' not found in pull request body, setting skip=false" - fi - - name: Check For Skip Condition - id: check_skip - run: | - if ${{ steps.check_commits.outputs.skip == 'true' }} || ${{ steps.check_pr_body.outputs.skip == 'true' }}; then - echo "skip=true" >> $GITHUB_OUTPUT - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - - build_and_test: - runs-on: ubuntu-latest - needs: check_commit_message - if: ${{ needs.check_commit_message.outputs.should_skip != 'true' }} - env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - steps: - - run: echo "${{ needs.check_commit_message.outputs.should_skip == 'true' }}" - - - name: Checkout code (origin/master) - uses: actions/checkout@v4 - with: - ref: master - fetch-depth: 0 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "backwards-compatibility" - - - name: Build iggy-server (origin/master) - run: IGGY_CI_BUILD=true cargo build - - - uses: JarvusInnovations/background-action@v1 - name: Run iggy-server in background (origin/master) - with: - run: | - target/debug/iggy-server & - wait-on: tcp:localhost:8090 - wait-for: 1m - log-output: true - log-output-if: timeout - tail: true - - - name: Run producer bench (origin/master) - timeout-minutes: 1 - run: target/debug/iggy-bench --verbose --message-batches 50 --messages-per-batch 100 pinned-producer tcp - - - name: Run consumer bench (origin/master) - timeout-minutes: 1 - run: target/debug/iggy-bench --verbose --message-batches 50 --messages-per-batch 100 pinned-consumer tcp - - - name: Stop iggy-server (origin/master) - timeout-minutes: 1 - run: pkill -15 iggy-server && while pgrep -l iggy-server; do sleep 2; done; - - - name: Print iggy-server logs (origin/master) - run: cat local_data/logs/iggy* - - - name: Remove iggy-server logs (origin/master) - run: rm local_data/logs/iggy* - - - name: Copy local_data directory (origin/master) - run: cp -r local_data .. - - - name: Checkout code (PR) - uses: actions/checkout@v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - fetch-depth: 0 - clean: false - - - name: Build iggy-server (PR) - run: IGGY_CI_BUILD=true cargo build - - - name: Restore local_data directory (PR) - run: cp -r ../local_data . - - - uses: JarvusInnovations/background-action@v1 - name: Run iggy-server in background (PR) - with: - run: | - target/debug/iggy-server & - wait-on: tcp:localhost:8090 - wait-for: 1m - log-output: true - log-output-if: timeout - tail: true - - - name: Run consumer bench (PR) - timeout-minutes: 1 - run: target/debug/iggy-bench --verbose --message-batches 50 --messages-per-batch 100 pinned-consumer tcp - - - name: Stop iggy-server (PR) - timeout-minutes: 1 - run: pkill iggy-server && while pgrep -l iggy-server; do sleep 1; done; - - - name: Print server logs (PR) - run: cat local_data/logs/iggy* diff --git a/.github/workflows/ci-coverage-rust.yml b/.github/workflows/ci-coverage-rust.yml deleted file mode 100644 index b7af211c5b..0000000000 --- a/.github/workflows/ci-coverage-rust.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Coverage Rust Workflow -# -# This workflow runs tests for Rust code and generates coverage report. -# This workflow can be triggered manually or by other workflows. -# -name: ci-coverage-rust - -on: - workflow_dispatch: - workflow_call: - -env: - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - GITHUB_BOT_CONTEXT_STRING: "coveralls coverage reporting job" - -jobs: - coverage: - name: coverage - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "coverage" - - - name: Install gnome-keyring, keyutils and lcov - run: | - sudo apt-get update --yes && sudo apt-get install --yes gnome-keyring keyutils lcov - rm -f $HOME/.local/share/keyrings/* - echo -n "test" | gnome-keyring-daemon --unlock - - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@v2 - with: - tool: cargo-llvm-cov - - - name: Build - run: source <(cargo llvm-cov show-env --export-prefix) && cargo build - - - name: Test - run: source <(cargo llvm-cov show-env --export-prefix) && cargo test - - - name: Generate code coverage - run: source <(cargo llvm-cov show-env --export-prefix) && cargo llvm-cov report --lcov --output-path coverage.lcov --ignore-filename-regex '(bench\/|integration\/|tools\/|tpc\/)' - - - name: Display code coverage - run: lcov --summary coverage.lcov - - - name: Upload code to Coveralls - # Do not upload coverage for user triggered workflows - if: ${{ github.event_name == 'workflow_call' }} - uses: coverallsapp/github-action@v2 - with: - fail-on-error: false - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-prod-rust.yml b/.github/workflows/ci-prod-rust.yml deleted file mode 100644 index d823bed605..0000000000 --- a/.github/workflows/ci-prod-rust.yml +++ /dev/null @@ -1,170 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Production Rust Workflow -# -# This workflow runs production actions for Rust code. -# It is triggered by changes to Rust files. It builds and -# iggy-server and iggy-cli binaries and creates Docker images -# for x86_64 and aarch64 architectures, then images are pushed to -# Docker Hub by digest. In the next step, the workflow creates -# edge tag based on the digests and pushes the manifest list -# to Docker Hub. -# -# This workflow can be triggered only oby other workflows. -# -name: ci-prod-rust - -on: - workflow_call: - secrets: - DOCKERHUB_USER: - required: true - DOCKERHUB_TOKEN: - required: true - -env: - DOCKERHUB_REGISTRY_NAME: apache/iggy - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - docker-edge-build: - name: ${{ matrix.platform.target }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - platform: - - target: x86_64-unknown-linux-musl - docker_arch: linux/amd64 - qemu: false - qemu_arch: "" - - target: aarch64-unknown-linux-musl - docker_arch: linux/arm64/v8 - qemu: true - qemu_arch: "arm64" - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "${{ matrix.platform.target }}" - - name: Install musl-tools, gnome-keyring and keyutils - run: | - sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils - rm -f $HOME/.local/share/keyrings/* - echo -n "test" | gnome-keyring-daemon --unlock - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Install cross tool - uses: taiki-e/install-action@v2 - with: - tool: cross - - name: Set verbose flag - shell: bash - run: echo "VERBOSE_FLAG=$([[ "${RUNNER_DEBUG}" = "1" ]] && echo "--verbose" || echo "")" >> $GITHUB_ENV - - name: Build iggy-server ${{ matrix.platform.target }} release binary - run: cross +stable build ${{ env.VERBOSE_FLAG }} --release --target ${{ matrix.platform.target }} --bin iggy-server - - name: Build iggy-cli ${{ matrix.platform.target }} release binary - run: cross +stable build ${{ env.VERBOSE_FLAG }} --release --no-default-features --target ${{ matrix.platform.target }} --bin iggy - - name: Print build ready message - run: echo "::notice ::Build binary artifacts for ${{ matrix.platform.target }}" - - name: Set up QEMU - if: ${{ matrix.platform.qemu == true }} - uses: docker/setup-qemu-action@v3 - with: - platforms: ${{ matrix.platform.qemu_arch }} - - name: Set up Docker - uses: docker/setup-docker-action@v4 - - name: Set up Docker buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.ci - platforms: ${{ matrix.platform.docker_arch }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.DOCKERHUB_REGISTRY_NAME }},push-by-digest=true,name-canonical=true - push: true - build-args: | - IGGY_CMD_PATH=target/${{ matrix.platform.target }}/release/iggy - IGGY_SERVER_PATH=target/${{ matrix.platform.target }}/release/iggy-server - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - ls -l /tmp/digests - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform.target }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - docker_edge_publish: - name: Docker edge tag publish - runs-on: ubuntu-latest - needs: docker-edge-build - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: "digests-*" - merge-multiple: true - path: /tmp/digests - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Create Docker meta - if: ${{ needs.tag.outputs.tag_created == '' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: edge - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKERHUB_REGISTRY_NAME }}@sha256:%s ' *) - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKERHUB_REGISTRY_NAME }}:edge diff --git a/.github/workflows/ci-prod.yml b/.github/workflows/ci-prod.yml deleted file mode 100644 index 04239ea9da..0000000000 --- a/.github/workflows/ci-prod.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Production Workflow -# -name: ci-prod - -on: - push: - branches: - - master - -jobs: - merge-file-changes: - name: merge-file-changes - runs-on: ubuntu-latest - outputs: - trigger-rust: ${{ steps.changed-files.outputs.RUST-PROD_FILES_CHANGED }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install python3 - run: | - sudo apt-get update - sudo apt-get install -y python3 - python3 --version - - name: Detect file changes - id: changed-files - run: | - .github/scripts/detect-changed-files.sh master - - name: List all changed files - run: | - if [ "${{ steps.changed-files.outputs.RUST-PROD_FILES_CHANGED }}" == "true" ]; then - echo "One or more rust file(s) has changed." - echo "List all the files that have changed: ${{ steps.changed-files.outputs.RUST-PROD_FILES }}" - fi - - ci-prod-rust: - name: ci-prod-rust - needs: merge-file-changes - if: ${{ needs.merge-file-changes.outputs.trigger-rust == 'true' }} - uses: ./.github/workflows/ci-prod-rust.yml - secrets: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - - finalize-prod: - runs-on: ubuntu-latest - needs: - - ci-prod-rust - if: always() - steps: - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Some tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 diff --git a/.github/workflows/ci-publish-go.yml b/.github/workflows/ci-publish-go.yml deleted file mode 100644 index 8b52a66a4e..0000000000 --- a/.github/workflows/ci-publish-go.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: ci-publish-go -on: - push: - tags: - - "foreign/go/v*" - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - if [[ ! "$TAG" =~ ^foreign/go/v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$ ]]; then - echo "Tag $TAG does not match required format: foreign/go/vX.Y.Z" - exit 1 - fi - echo "Valid Go module tag: $TAG" - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Check go.mod module path in foreign/go - run: | - MOD_PATH=$(grep '^module ' foreign/go/go.mod | awk '{print $2}') - if [[ "$MOD_PATH" != "github.com/${{ github.repository }}/foreign/go" ]]; then - echo "go.mod module path ($MOD_PATH) does not match expected path: github.com/${{ github.repository }}/foreign/go" - exit 1 - fi - echo "go.mod module path is correct: $MOD_PATH" - - - name: Check if go.mod version matches tag - run: | - TAG_VERSION="${TAG#foreign/go/}" - # Check if go.mod contains this version in replace or require (not strictly necessary, but can be used for extra checks) - echo "Tag version is $TAG_VERSION (no strict version field in go.mod to check for single module)" - # (No error here, just informative) - - - name: Run go mod tidy - working-directory: foreign/go - run: | - go mod tidy - git diff --exit-code || (echo 'Please run "go mod tidy" and commit the changes before tagging.' && exit 1) - - - name: Run Go tests - working-directory: foreign/go - run: go test -v ./... - diff --git a/.github/workflows/ci-test-bdd.yml b/.github/workflows/ci-test-bdd.yml deleted file mode 100644 index 2f024d04c7..0000000000 --- a/.github/workflows/ci-test-bdd.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Cross-SDK BDD Tests - -on: - workflow_call: - -jobs: - bdd-tests: - name: BDD tests (${{ matrix.sdk }}) - runs-on: ubuntu-latest - strategy: - matrix: - sdk: [rust, python, go, node, csharp] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - . -> target - foreign/python -> target - cache-on-failure: true - key: bdd-${{ matrix.sdk }} - - - name: Run BDD tests for ${{ matrix.sdk }} - run: ./scripts/run-bdd-tests.sh ${{ matrix.sdk }} - - - name: Clean up Docker resources - if: always() - run: ./scripts/run-bdd-tests.sh clean diff --git a/.github/workflows/ci-test-rust-optional.yml b/.github/workflows/ci-test-rust-optional.yml deleted file mode 100644 index cff1e5e1ec..0000000000 --- a/.github/workflows/ci-test-rust-optional.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Test Rust Optional Workflow -# -# This workflow runs tests for Rust code on aarch64-apple-darwin target and generates coverage report. -# This workflow is optional and can be triggered manually or by other workflows. -# -name: ci-test-rust-optional - -on: - workflow_dispatch: - workflow_call: - -env: - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - ci-coverage-rust: - name: ci-coverage-rust - uses: ./.github/workflows/ci-coverage-rust.yml - - aarch64-apple-darwin: - name: aarch64-apple-darwin - runs-on: macos-latest - timeout-minutes: 30 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "aarch64-apple-darwin" - - name: Prepare aarch64-apple-darwin toolchain - run: | - rustup target add aarch64-apple-darwin - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Set verbose flag - shell: bash - run: echo "VERBOSE_FLAG=$([[ "${RUNNER_DEBUG}" = "1" ]] && echo "--verbose" || echo "")" >> $GITHUB_ENV - - name: Build binary - run: cargo build ${{ env.VERBOSE_FLAG }} --target aarch64-apple-darwin - - name: Run tests - run: cargo test ${{ env.VERBOSE_FLAG }} --target aarch64-apple-darwin - - name: Check if workspace is clean - run: git status --porcelain diff --git a/.github/workflows/ci-test-rust.yml b/.github/workflows/ci-test-rust.yml deleted file mode 100644 index 39ba91a234..0000000000 --- a/.github/workflows/ci-test-rust.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# ------------------------------------------------------------- -# -# CI Test Rust Workflow -# -# This workflow runs tests for Rust code. -# All tests are run on multiple targets: -# - x86_64-unknown-linux-musl -# - x86_64-unknown-linux-gnu -# - x86_64-pc-windows-msvc -# -# This workflow can be triggered manually or by other workflows. -# -name: ci-test-rust - -on: - workflow_dispatch: - workflow_call: - -env: - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - x86_64-unknown-linux: - name: ${{ matrix.target }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - target: [x86_64-unknown-linux-musl, x86_64-unknown-linux-gnu] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "${{ matrix.target }}" - - name: Install musl-tools, gnome-keyring and keyutils - run: | - sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils - rm -f $HOME/.local/share/keyrings/* - echo -n "test" | gnome-keyring-daemon --unlock - - name: Prepare ${{ matrix.target }} toolchain - run: | - rustup target add ${{ matrix.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Set verbose flag - shell: bash - run: echo "VERBOSE_FLAG=$([[ "${RUNNER_DEBUG}" = "1" ]] && echo "--verbose" || echo "")" >> $GITHUB_ENV - - name: Build binary ${{ matrix.target }} - run: cargo build ${{ env.VERBOSE_FLAG }} --target ${{ matrix.target }} - - name: Run tests ${{ matrix.target }} - run: cargo test ${{ env.VERBOSE_FLAG }} --target ${{ matrix.target }} - - name: Check CLI examples from README - run: ./scripts/run-rust-examples-from-readme.sh ${{ matrix.target }} - - name: Check if workspace is clean - run: git status --porcelain - - x86_64-pc-windows-msvc: - name: x86_64-pc-windows-msvc - runs-on: windows-latest - timeout-minutes: 30 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "x86_64-pc-windows-msvc" - - name: Prepare x86_64-pc-windows-msvc toolchain - run: | - rustup target add x86_64-pc-windows-msvc - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - name: Set verbose flag - shell: pwsh - run: | - if ($env:RUNNER_DEBUG -eq "1") { - echo "VERBOSE_FLAG=--verbose" >> $env:GITHUB_ENV - } else { - echo "VERBOSE_FLAG=" >> $env:GITHUB_ENV - } - - name: Build iggy package - run: cargo build ${{ env.VERBOSE_FLAG }} --target x86_64-pc-windows-msvc -p iggy - - name: Build iggy-cli binary - run: cargo build ${{ env.VERBOSE_FLAG }} --target x86_64-pc-windows-msvc --bin iggy - - name: Check if workspace is clean - run: git status --porcelain diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml new file mode 100644 index 0000000000..03a83295e8 --- /dev/null +++ b/.github/workflows/post-merge.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Post-merge + +on: + push: + branches: [master] + +permissions: + contents: read + packages: write + id-token: write + +concurrency: + group: post-merge-${{ github.ref }} + cancel-in-progress: false + +env: + IGGY_CI_BUILD: true + +jobs: + plan: + name: Plan dockerhub components + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.mk.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Load publish config (base64) + id: cfg + shell: bash + run: | + if ! command -v yq >/dev/null 2>&1; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 + chmod +x /usr/local/bin/yq + fi + echo "components_b64=$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" >> "$GITHUB_OUTPUT" + + - name: Build matrix + id: mk + uses: actions/github-script@v7 + with: + script: | + const b64 = `${{ steps.cfg.outputs.components_b64 }}` || ''; + if (!b64) { + core.setOutput('matrix', JSON.stringify({ include: [{ component: 'noop' }] })); + return; + } + const comps = JSON.parse(Buffer.from(b64, 'base64').toString('utf8')); + const include = Object.entries(comps) + .filter(([_, v]) => v && v.registry === 'dockerhub') + .map(([k]) => ({ component: k })); + const uniq = Array.from(new Map(include.map(i => [i.component, i])).values()); + core.setOutput('matrix', JSON.stringify(uniq.length ? { include: uniq } : { include: [{ component: 'noop' }] })); + + docker-edge: + name: ${{ matrix.component }} + needs: plan + if: ${{ fromJson(needs.plan.outputs.matrix).include[0].component != 'noop' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.matrix) }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/utils/docker-buildx + with: + task: publish + libc: musl + component: ${{ matrix.component }} + version: edge + dry_run: ${{ github.event.repository.fork }} # forks: always dry-run diff --git a/.github/workflows/post_publish_server.yml b/.github/workflows/post_publish_server.yml deleted file mode 100644 index d82ac7b0fb..0000000000 --- a/.github/workflows/post_publish_server.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: post_publish_server -on: - workflow_dispatch: - workflow_run: - workflows: [ "publish_server" ] - types: - - completed - -jobs: - post_publish_server: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'}} - steps: - - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "post-publish-server" - - - name: Build binary - run: cargo build - - - uses: addnab/docker-run-action@v3 - name: Spin up Docker Container - with: - image: apache/iggy:latest - options: -d -p 8090:8090 - run: /iggy/iggy-server - - - name: Wait till iggy-server has bound to TCP 8090 port - timeout-minutes: 1 - run: | - while ! nc -z 127.0.0.1 8090; do - sleep 1 - done - sleep 1 - ss -tuln | grep :8090 - - - name: Test Benchmark - Producer - timeout-minutes: 1 - run: | - ./target/debug/iggy-bench --skip-server-start --message-batches 100 --messages-per-batch 100 pinned-producer tcp --server-address 127.0.0.1:8090 - - - name: Test Benchmark - Consumer - timeout-minutes: 1 - run: | - ./target/debug/iggy-bench --skip-server-start --message-batches 100 --messages-per-batch 100 pinned-consumer tcp --server-address 127.0.0.1:8090 - - - name: Test Benchmark - Producer and Consumer - timeout-minutes: 1 - run: | - ./target/debug/iggy-bench --skip-server-start --message-batches 100 --messages-per-batch 100 pinned-producer-and-consumer tcp --server-address 127.0.0.1:8090 - - - name: Check if number of messages is correct - timeout-minutes: 1 - run: | - STATS=$(./target/debug/iggy -u iggy -p iggy stats) - echo "$STATS" - MESSAGE_COUNT=$(./target/debug/iggy -u iggy -p iggy -q stats -o json | jq '.messages_count') - readonly EXPECTED_MESSAGE_COUNT=160000 - if [ "$MESSAGE_COUNT" -ne "$EXPECTED_MESSAGE_COUNT" ]; then - echo "Expected message count to be $EXPECTED_MESSAGE_COUNT, but got $MESSAGE_COUNT" - exit 1 - fi - - - name: Clean up - run: docker rm -f iggy_container - - finalize_post_publish_server: - runs-on: ubuntu-latest - needs: [ post_publish_server ] - if: always() - steps: - - uses: actions/checkout@v4 - - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name == 'workflow_run' }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "post release docker container test" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml new file mode 100644 index 0000000000..5364afe6f8 --- /dev/null +++ b/.github/workflows/pre-merge.yml @@ -0,0 +1,426 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Pre-merge +on: + pull_request: + branches: [master] + # TODO(hubcio): remove push, no need to do it + push: + branches: [master] + workflow_dispatch: + +env: + IGGY_CI_BUILD: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +permissions: + contents: read + security-events: write + pull-requests: read + +jobs: + # Common checks - always run + common: + name: Common checks + uses: ./.github/workflows/_common.yml + permissions: + contents: read + pull-requests: read + + # Detect changes and build matrices + detect: + name: Detect changes + uses: ./.github/workflows/_detect.yml + + # Rust components + test-rust: + name: Rust β€’ ${{ matrix.component }}/${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.rust_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.rust_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Python SDK + test-python: + name: Python β€’ ${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.python_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.python_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Node SDK + test-node: + name: Node β€’ ${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.node_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.node_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Go SDK + test-go: + name: Go β€’ ${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.go_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.go_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Java SDK + test-java: + name: Java β€’ ${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.java_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.java_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # C# SDK + test-csharp: + name: C# β€’ ${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.csharp_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.csharp_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Other components + test-other: + name: Other β€’ ${{ matrix.component }}/${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.other_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.other_matrix) }} + uses: ./.github/workflows/_test.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # BDD Tests + test-bdd: + name: BDD β€’ ${{ matrix.component }}/${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.bdd_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.bdd_matrix) }} + uses: ./.github/workflows/_test_bdd.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Examples Tests + test-examples: + name: Examples β€’ ${{ matrix.component }}/${{ matrix.task }} + needs: detect + if: ${{ fromJson(needs.detect.outputs.examples_matrix).include[0].component != 'noop' }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect.outputs.examples_matrix) }} + uses: ./.github/workflows/_test_examples.yml + with: + component: ${{ matrix.component }} + task: ${{ matrix.task }} + + # Final status check + status: + name: CI Status + runs-on: ubuntu-latest + needs: [common, detect, test-rust, test-python, test-node, test-go, test-java, test-csharp, test-bdd, test-examples, test-other] + if: always() + steps: + - name: Get job execution times + id: times + uses: actions/github-script@v7 + with: + script: | + const jobs = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId + }); + + const jobTimes = {}; + const jobStatuses = {}; + + const formatDuration = (ms) => { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + if (hours > 0) { + return `${hours}h ${minutes % 60}m ${seconds % 60}s`; + } else if (minutes > 0) { + return `${minutes}m ${seconds % 60}s`; + } else if (seconds > 0) { + return `${seconds}s`; + } else { + return '< 1s'; + } + }; + + // Log job names for debugging + console.log('Job names found:'); + for (const job of jobs.data.jobs) { + console.log(` - ${job.name}: ${job.status} (conclusion: ${job.conclusion || 'N/A'})`); + jobStatuses[job.name] = job.conclusion || job.status; + + // Only show duration for jobs that actually ran + if (job.started_at && job.completed_at) { + const start = new Date(job.started_at); + const end = new Date(job.completed_at); + const duration = end - start; + jobTimes[job.name] = formatDuration(duration); + } else if (job.started_at && !job.completed_at) { + // Job is still running + const start = new Date(job.started_at); + const duration = Date.now() - start; + jobTimes[job.name] = formatDuration(duration) + ' ⏳'; + } else { + // Job was skipped or hasn't started + jobTimes[job.name] = null; + } + } + + // Helper to find job info for a component + const findJobInfo = (prefix) => { + for (const [name, status] of Object.entries(jobStatuses)) { + if (name.startsWith(prefix)) { + return { + time: jobTimes[name], + status: status + }; + } + } + return { time: null, status: 'skipped' }; + }; + + // Format duration based on job status + const formatJobDuration = (info) => { + if (info.status === 'skipped') return '-'; + if (info.time === null) return '-'; + return info.time; + }; + + // Set outputs for each component + const rust = findJobInfo('Rust β€’'); + const python = findJobInfo('Python β€’'); + const node = findJobInfo('Node β€’'); + const go = findJobInfo('Go β€’'); + const java = findJobInfo('Java β€’'); + const csharp = findJobInfo('C# β€’'); + const bdd = findJobInfo('BDD β€’'); + const examples = findJobInfo('Examples β€’'); + const other = findJobInfo('Other β€’'); + + // For non-matrix jobs, check by exact name + const common = jobStatuses['Common checks'] + ? { time: jobTimes['Common checks'], status: jobStatuses['Common checks'] } + : { time: null, status: 'skipped' }; + const detect = jobStatuses['Detect changes'] + ? { time: jobTimes['Detect changes'], status: jobStatuses['Detect changes'] } + : { time: null, status: 'skipped' }; + + // Output formatted durations + core.setOutput('rust_time', formatJobDuration(rust)); + core.setOutput('python_time', formatJobDuration(python)); + core.setOutput('node_time', formatJobDuration(node)); + core.setOutput('go_time', formatJobDuration(go)); + core.setOutput('java_time', formatJobDuration(java)); + core.setOutput('csharp_time', formatJobDuration(csharp)); + core.setOutput('bdd_time', formatJobDuration(bdd)); + core.setOutput('examples_time', formatJobDuration(examples)); + core.setOutput('other_time', formatJobDuration(other)); + core.setOutput('common_time', formatJobDuration(common)); + core.setOutput('detect_time', formatJobDuration(detect)); + + // Calculate total time - find the earliest job start time + let earliestStart = null; + let latestEnd = null; + for (const job of jobs.data.jobs) { + if (job.started_at) { + const start = new Date(job.started_at); + if (!earliestStart || start < earliestStart) { + earliestStart = start; + } + } + if (job.completed_at) { + const end = new Date(job.completed_at); + if (!latestEnd || end > latestEnd) { + latestEnd = end; + } + } + } + + if (earliestStart && latestEnd) { + const totalDuration = latestEnd - earliestStart; + core.setOutput('total_time', formatDuration(totalDuration)); + } else if (earliestStart) { + const totalDuration = Date.now() - earliestStart; + core.setOutput('total_time', formatDuration(totalDuration) + ' (running)'); + } else { + core.setOutput('total_time', '-'); + } + + return jobTimes; + + - name: Check status + run: | + set -euxo pipefail + echo "## CI Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Helper function to format status with appropriate emoji + format_status() { + local status="$1" + local duration="$2" + + case "$status" in + "success") + if [[ "$duration" == "-" ]]; then + echo "βœ… success" + else + echo "βœ… success" + fi + ;; + "failure") + echo "❌ failure" + ;; + "cancelled") + echo "🚫 cancelled" + ;; + "skipped") + echo "⏭️ skipped" + ;; + *) + echo "⏸️ $status" + ;; + esac + } + + # Language test results with timing + echo "### Test Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Component | Status | Duration |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|--------|----------|" >> $GITHUB_STEP_SUMMARY + + # Detection and Common checks always run + detect_status=$(format_status "${{ needs.detect.result }}" "${{ steps.times.outputs.detect_time }}") + common_status=$(format_status "${{ needs.common.result }}" "${{ steps.times.outputs.common_time }}") + echo "| πŸ” Detection | $detect_status | ${{ steps.times.outputs.detect_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| πŸ“‹ Common Checks | $common_status | ${{ steps.times.outputs.common_time }} |" >> $GITHUB_STEP_SUMMARY + + # Language/component tests + rust_status=$(format_status "${{ needs.test-rust.result }}" "${{ steps.times.outputs.rust_time }}") + python_status=$(format_status "${{ needs.test-python.result }}" "${{ steps.times.outputs.python_time }}") + node_status=$(format_status "${{ needs.test-node.result }}" "${{ steps.times.outputs.node_time }}") + go_status=$(format_status "${{ needs.test-go.result }}" "${{ steps.times.outputs.go_time }}") + java_status=$(format_status "${{ needs.test-java.result }}" "${{ steps.times.outputs.java_time }}") + csharp_status=$(format_status "${{ needs.test-csharp.result }}" "${{ steps.times.outputs.csharp_time }}") + bdd_status=$(format_status "${{ needs.test-bdd.result }}" "${{ steps.times.outputs.bdd_time }}") + examples_status=$(format_status "${{ needs.test-examples.result }}" "${{ steps.times.outputs.examples_time }}") + other_status=$(format_status "${{ needs.test-other.result }}" "${{ steps.times.outputs.other_time }}") + + echo "| πŸ¦€ Rust | $rust_status | ${{ steps.times.outputs.rust_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| 🐍 Python | $python_status | ${{ steps.times.outputs.python_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| 🟒 Node | $node_status | ${{ steps.times.outputs.node_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| 🐹 Go | $go_status | ${{ steps.times.outputs.go_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| β˜• Java | $java_status | ${{ steps.times.outputs.java_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| πŸ”· C# | $csharp_status | ${{ steps.times.outputs.csharp_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| πŸ§ͺ BDD | $bdd_status | ${{ steps.times.outputs.bdd_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| πŸ“š Examples | $examples_status | ${{ steps.times.outputs.examples_time }} |" >> $GITHUB_STEP_SUMMARY + echo "| πŸ“¦ Other | $other_status | ${{ steps.times.outputs.other_time }} |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Total workflow time:** ${{ steps.times.outputs.total_time }}" >> $GITHUB_STEP_SUMMARY + + # Check for failures + if [[ "${{ needs.common.result }}" == "failure" ]] || \ + [[ "${{ needs.detect.result }}" == "failure" ]] || \ + [[ "${{ needs.test-rust.result }}" == "failure" ]] || \ + [[ "${{ needs.test-python.result }}" == "failure" ]] || \ + [[ "${{ needs.test-node.result }}" == "failure" ]] || \ + [[ "${{ needs.test-go.result }}" == "failure" ]] || \ + [[ "${{ needs.test-java.result }}" == "failure" ]] || \ + [[ "${{ needs.test-csharp.result }}" == "failure" ]] || \ + [[ "${{ needs.test-bdd.result }}" == "failure" ]] || \ + [[ "${{ needs.test-examples.result }}" == "failure" ]] || \ + [[ "${{ needs.test-other.result }}" == "failure" ]]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "❌ **CI Failed** - Please check the logs for details." >> $GITHUB_STEP_SUMMARY + exit 1 + elif [[ "${{ needs.common.result }}" == "cancelled" ]] || \ + [[ "${{ needs.detect.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-rust.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-python.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-node.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-go.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-java.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-csharp.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-bdd.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-examples.result }}" == "cancelled" ]] || \ + [[ "${{ needs.test-other.result }}" == "cancelled" ]]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **CI Cancelled** - The workflow was cancelled." >> $GITHUB_STEP_SUMMARY + exit 1 + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **CI Passed** - All checks completed successfully!" >> $GITHUB_STEP_SUMMARY + fi + + finalize_pr: + runs-on: ubuntu-latest + needs: + - status + if: always() + steps: + - name: Everything is fine + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..162d48c735 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,947 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Publish + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run (build/test only, no actual publish)" + type: boolean + default: true + use_latest_ci: + description: "Use latest CI configuration and scripts from master branch (recommended for compatibility)" + type: boolean + required: false + default: true + commit: + description: "Commit SHA to publish from" + type: string + required: true + publish_crates: + description: "Rust crates to publish (comma-separated: rust-sdk, rust-cli, rust-binary-protocol, rust-common)" + type: string + required: false + default: "" + publish_dockerhub: + description: "Docker images to publish (comma-separated: rust-server, rust-mcp, rust-bench-dashboard, rust-connectors, web-ui)" + type: string + required: false + default: "" + publish_other: + description: "Other SDKs to publish (comma-separated: python, node, java, csharp, go:VERSION)" + type: string + required: false + default: "" + +env: + IGGY_CI_BUILD: true + +permissions: + contents: write # For tag creation + packages: write + id-token: write + +concurrency: + group: publish-${{ github.run_id }} + cancel-in-progress: false + +jobs: + validate: + name: Validate inputs + runs-on: ubuntu-latest + outputs: + commit: ${{ steps.resolve.outputs.commit }} + has_targets: ${{ steps.check.outputs.has_targets }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if any targets specified + id: check + run: | + if [ -z "${{ inputs.publish_crates }}" ] && \ + [ -z "${{ inputs.publish_dockerhub }}" ] && \ + [ -z "${{ inputs.publish_other }}" ]; then + echo "has_targets=false" >> "$GITHUB_OUTPUT" + else + echo "has_targets=true" >> "$GITHUB_OUTPUT" + fi + + - name: Resolve commit + id: resolve + run: | + COMMIT="${{ inputs.commit }}" + if [ -z "$COMMIT" ]; then + echo "❌ No commit specified" + exit 1 + fi + + if ! git rev-parse --verify "$COMMIT^{commit}" >/dev/null 2>&1; then + echo "❌ Invalid commit: $COMMIT" + exit 1 + fi + + # Verify commit is on master branch + echo "πŸ” Verifying commit is on master branch..." + git fetch origin master --depth=1000 + + if git merge-base --is-ancestor "$COMMIT" origin/master; then + echo "βœ… Commit is on master branch" + else + echo "❌ ERROR: Commit $COMMIT is not on the master branch!" + echo "" + echo "Publishing is only allowed from commits on the master branch." + echo "Please ensure your commit has been merged to master before publishing." + echo "" + echo "To check which branch contains this commit, run:" + echo " git branch -r --contains $COMMIT" + exit 1 + fi + + echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" + echo "βœ… Will publish from commit: $COMMIT" + echo + echo "Commit details:" + git log -1 --pretty=format:" Author: %an <%ae>%n Date: %ad%n Subject: %s" "$COMMIT" + + plan: + name: Build publish plan + needs: validate + if: needs.validate.outputs.has_targets == 'true' + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.mk.outputs.targets }} + non_rust_targets: ${{ steps.mk.outputs.non_rust_targets }} + count: ${{ steps.mk.outputs.count }} + go_sdk_version: ${{ steps.mk.outputs.go_sdk_version }} + has_python: ${{ steps.mk.outputs.has_python }} + has_rust_crates: ${{ steps.mk.outputs.has_rust_crates }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + + - name: Load publish config + id: cfg + run: | + if ! command -v yq &> /dev/null; then + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + wget -qO /tmp/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /tmp/yq" | sha256sum -c - || exit 1 + chmod +x /tmp/yq && sudo mv /tmp/yq /usr/local/bin/yq + fi + echo "components_b64=$(yq -o=json -I=0 '.components' .github/config/publish.yml | base64 -w0)" >> "$GITHUB_OUTPUT" + + - name: Build matrix from inputs + id: mk + uses: actions/github-script@v7 + with: + script: | + const componentsB64 = '${{ steps.cfg.outputs.components_b64 }}'; + const cfg = JSON.parse(Buffer.from(componentsB64, 'base64').toString('utf-8') || "{}"); + + const wants = []; + let goVersion = ''; + + // Parse Rust crates + ('${{ inputs.publish_crates }}').split(',').map(s => s.trim()).filter(Boolean).forEach(crate => { + if (['rust-sdk','rust-cli','rust-binary-protocol','rust-common'].includes(crate)) wants.push(crate); + else core.warning(`Unknown crate: ${crate}`); + }); + + // Parse Docker images + ('${{ inputs.publish_dockerhub }}').split(',').map(s => s.trim()).filter(Boolean).forEach(img => { + if (['rust-server','rust-mcp','rust-bench-dashboard','rust-connectors','web-ui'].includes(img)) wants.push(img); + else core.warning(`Unknown Docker image: ${img}`); + }); + + // Parse other SDKs + ('${{ inputs.publish_other }}').split(',').map(s => s.trim()).filter(Boolean).forEach(sdk => { + if (sdk.startsWith('go:')) { + goVersion = sdk.substring(3); + if (!/^\d+\.\d+\.\d+/.test(goVersion)) { + core.setFailed(`Invalid Go version format: ${goVersion} (expected: X.Y.Z)`); + } else { + wants.push('sdk-go'); + } + } else if (['python','node','java','csharp'].includes(sdk)) { + wants.push(`sdk-${sdk}`); + } else { + core.warning(`Unknown SDK: ${sdk}`); + } + }); + + const toType = (entry) => ({ + dockerhub: 'docker', + crates: 'rust', + pypi: 'python', + npm: 'node', + maven: 'java', + nuget: 'csharp', + none: 'go' + }[entry.registry] || 'unknown'); + + const targets = []; + const nonRustTargets = []; + const seen = new Set(); + let hasRustCrates = false; + + for (const key of wants) { + if (seen.has(key)) continue; + seen.add(key); + + const entry = cfg[key]; + if (!entry) { core.warning(`Component '${key}' not found in publish.yml`); continue; } + + const target = { + key, + name: key, + type: toType(entry), + registry: entry.registry || '', + package: entry.package || '', + image: entry.image || '', + dockerfile: entry.dockerfile || '', + platforms: Array.isArray(entry.platforms) ? entry.platforms.join(',') : '', + tag_pattern: entry.tag_pattern || '', + version_file: entry.version_file || '', + version_regex: entry.version_regex || '' + }; + + targets.push(target); + + // Separate Rust crates from other targets + if (target.type === 'rust') { + hasRustCrates = true; + // Rust crates are handled by the sequential job + } else { + nonRustTargets.push(target); + } + } + + console.log(`Publishing ${targets.length} components:`); + targets.forEach(t => console.log(` - ${t.name} (${t.type}) -> ${t.registry || 'N/A'}`)); + console.log(` (${nonRustTargets.length} non-Rust, ${targets.length - nonRustTargets.length} Rust crates)`); + + // Output all targets for reference and tag creation + core.setOutput('targets', JSON.stringify(targets.length ? { include: targets } : { include: [{ key: 'noop', type: 'noop' }] })); + + // Output only non-Rust targets for the parallel publish job + core.setOutput('non_rust_targets', JSON.stringify(nonRustTargets.length ? { include: nonRustTargets } : { include: [{ key: 'noop', type: 'noop' }] })); + + core.setOutput('count', String(targets.length)); + core.setOutput('go_sdk_version', goVersion); + core.setOutput('has_rust_crates', String(hasRustCrates)); + + // Check if Python SDK is in targets and extract version + const pythonTarget = targets.find(t => t.key === 'sdk-python'); + if (pythonTarget) { + core.setOutput('has_python', 'true'); + // Python version will be extracted in the publish job + } else { + core.setOutput('has_python', 'false'); + } + + check-tags: + name: Check existing tags + needs: [validate, plan] + if: needs.validate.outputs.has_targets == 'true' && fromJson(needs.plan.outputs.targets).include[0].key != 'noop' + runs-on: ubuntu-latest + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + echo "Scripts:" + ls -la /tmp/latest-scripts/ + echo ".github:" + ls -la /tmp/latest-github/ + + - name: Checkout at commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + fetch-depth: 0 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ + cp -r /tmp/latest-github/* .github/ + chmod +x scripts/*.sh + echo "βœ… Latest CI and scripts applied" + + - name: Setup yq + run: | + YQ_VERSION="v4.47.1" + YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb" + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1 + sudo chmod +x /usr/local/bin/yq + + - name: Check for existing tags + run: | + set -euo pipefail + + echo "## 🏷️ Tag Existence Check" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + TARGETS_JSON='${{ needs.plan.outputs.targets }}' + GO_SDK_VERSION='${{ needs.plan.outputs.go_sdk_version }}' + + EXISTING_TAGS=() + NEW_TAGS=() + + echo "| Component | Version | Tag | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|---------|-----|--------|" >> $GITHUB_STEP_SUMMARY + + echo "$TARGETS_JSON" | jq -r '.include[] | select(.key!="noop") | @base64' | while read -r row; do + _jq() { echo "$row" | base64 -d | jq -r "$1"; } + + KEY=$(_jq '.key') + NAME=$(_jq '.name') + TAG_PATTERN=$(_jq '.tag_pattern') + + # Skip components without tag patterns + if [ -z "$TAG_PATTERN" ] || [ "$TAG_PATTERN" = "null" ]; then + echo "Skipping $NAME - no tag pattern defined" + continue + fi + + # Extract version + GO_FLAG="" + if [ "$KEY" = "sdk-go" ] && [ -n "$GO_SDK_VERSION" ]; then + GO_FLAG="--go-sdk-version $GO_SDK_VERSION" + fi + + # Make script executable if needed + chmod +x scripts/extract-version.sh || true + + VERSION=$(scripts/extract-version.sh "$KEY" $GO_FLAG 2>/dev/null || echo "ERROR") + TAG=$(scripts/extract-version.sh "$KEY" $GO_FLAG --tag 2>/dev/null || echo "ERROR") + + if [ "$VERSION" = "ERROR" ] || [ "$TAG" = "ERROR" ]; then + echo "❌ Failed to extract version/tag for $NAME" + echo "| $NAME | ERROR | ERROR | ❌ Failed to extract |" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + # Check if tag exists + if git rev-parse "$TAG" >/dev/null 2>&1; then + EXISTING_TAGS+=("$TAG") + COMMIT_SHA=$(git rev-parse "$TAG" | head -c 8) + echo "⚠️ Tag exists: $TAG (points to $COMMIT_SHA)" + echo "| $NAME | $VERSION | $TAG | ⚠️ Exists at $COMMIT_SHA |" >> $GITHUB_STEP_SUMMARY + else + NEW_TAGS+=("$TAG") + echo "βœ… Tag will be created: $TAG" + echo "| $NAME | $VERSION | $TAG | βœ… Will create |" >> $GITHUB_STEP_SUMMARY + fi + done + + echo "" >> $GITHUB_STEP_SUMMARY + + # Summary + if [ ${#EXISTING_TAGS[@]} -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "### ⚠️ Warning: Existing Tags Detected" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The following tags already exist and will be skipped:" >> $GITHUB_STEP_SUMMARY + for tag in "${EXISTING_TAGS[@]}"; do + echo "- $tag" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ inputs.dry_run }}" = "false" ]; then + echo "**These components will NOT be republished.** Tags are immutable in git." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "If you need to republish:" >> $GITHUB_STEP_SUMMARY + echo "1. Delete the existing tag: \`git push --delete origin \`" >> $GITHUB_STEP_SUMMARY + echo "2. Bump the version in the source file" >> $GITHUB_STEP_SUMMARY + echo "3. Run the publish workflow again" >> $GITHUB_STEP_SUMMARY + fi + fi + + if [ ${#NEW_TAGS[@]} -eq 0 ] && [ ${#EXISTING_TAGS[@]} -gt 0 ]; then + echo "### ℹ️ No New Tags to Create" >> $GITHUB_STEP_SUMMARY + echo "All specified components have already been tagged. Consider bumping versions if you need to publish new releases." >> $GITHUB_STEP_SUMMARY + elif [ ${#NEW_TAGS[@]} -gt 0 ]; then + echo "### βœ… Tags to be Created" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + for tag in "${NEW_TAGS[@]}"; do + echo "- $tag" >> $GITHUB_STEP_SUMMARY + done + fi + + build-python-wheels: + name: Build Python wheels + needs: [validate, plan, check-tags] + if: | + needs.validate.outputs.has_targets == 'true' && + needs.plan.outputs.has_python == 'true' + uses: ./.github/workflows/_build_python_wheels.yml + with: + upload_artifacts: true + use_latest_ci: ${{ inputs.use_latest_ci }} + + # Sequential Rust crate publishing to handle dependencies properly + publish-rust-crates: + name: Publish Rust Crates + needs: [validate, plan, check-tags] + if: | + needs.validate.outputs.has_targets == 'true' && + contains(inputs.publish_crates, 'rust-') + runs-on: ubuntu-latest + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + DRY_RUN: ${{ inputs.dry_run }} + outputs: + status: ${{ steps.final-status.outputs.status }} + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + + - name: Checkout at commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + fetch-depth: 0 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ + cp -r /tmp/latest-github/* .github/ + chmod +x scripts/*.sh + echo "βœ… Latest CI and scripts applied" + + - name: Setup Rust with cache + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false + show-stats: false + + - name: Extract versions + id: versions + run: | + # Extract version for each crate + chmod +x scripts/extract-version.sh + + echo "common_version=$(scripts/extract-version.sh rust-common)" >> $GITHUB_OUTPUT + echo "protocol_version=$(scripts/extract-version.sh rust-binary-protocol)" >> $GITHUB_OUTPUT + echo "sdk_version=$(scripts/extract-version.sh rust-sdk)" >> $GITHUB_OUTPUT + echo "cli_version=$(scripts/extract-version.sh rust-cli)" >> $GITHUB_OUTPUT + + # Step 1: Publish iggy_common first + - name: Publish iggy_common + if: contains(inputs.publish_crates, 'rust-common') + uses: ./.github/actions/rust/post-merge + with: + package: iggy_common + version: ${{ steps.versions.outputs.common_version }} + dry_run: ${{ inputs.dry_run }} + + # Wait for crates.io to index (only in non-dry-run mode) + - name: Wait for iggy_common to be available + if: | + contains(inputs.publish_crates, 'rust-common') && + inputs.dry_run == 'false' + run: | + echo "⏳ Waiting for iggy_common to be available on crates.io..." + for i in {1..30}; do + if cargo search iggy_common --limit 1 | grep -q "^iggy_common = \"${{ steps.versions.outputs.common_version }}\""; then + echo "βœ… iggy_common is now available" + break + fi + echo "Waiting... (attempt $i/30)" + sleep 10 + done + + # Step 2: Publish iggy_binary_protocol (depends on common) + - name: Publish iggy_binary_protocol + if: contains(inputs.publish_crates, 'rust-binary-protocol') + uses: ./.github/actions/rust/post-merge + with: + package: iggy_binary_protocol + version: ${{ steps.versions.outputs.protocol_version }} + dry_run: ${{ inputs.dry_run }} + + # Wait for crates.io to index + - name: Wait for iggy_binary_protocol to be available + if: | + contains(inputs.publish_crates, 'rust-binary-protocol') && + inputs.dry_run == 'false' + run: | + echo "⏳ Waiting for iggy_binary_protocol to be available on crates.io..." + for i in {1..30}; do + if cargo search iggy_binary_protocol --limit 1 | grep -q "^iggy_binary_protocol = \"${{ steps.versions.outputs.protocol_version }}\""; then + echo "βœ… iggy_binary_protocol is now available" + break + fi + echo "Waiting... (attempt $i/30)" + sleep 10 + done + + # Step 3: Publish iggy SDK (depends on common and protocol) + - name: Publish iggy SDK + if: contains(inputs.publish_crates, 'rust-sdk') + uses: ./.github/actions/rust/post-merge + with: + package: iggy + version: ${{ steps.versions.outputs.sdk_version }} + dry_run: ${{ inputs.dry_run }} + + # Wait for crates.io to index + - name: Wait for iggy SDK to be available + if: | + contains(inputs.publish_crates, 'rust-sdk') && + inputs.dry_run == 'false' + run: | + echo "⏳ Waiting for iggy to be available on crates.io..." + for i in {1..30}; do + if cargo search iggy --limit 1 | grep -q "^iggy = \"${{ steps.versions.outputs.sdk_version }}\""; then + echo "βœ… iggy SDK is now available" + break + fi + echo "Waiting... (attempt $i/30)" + sleep 10 + done + + # Step 4: Publish iggy-cli (depends on SDK and protocol) + - name: Publish iggy-cli + if: contains(inputs.publish_crates, 'rust-cli') + uses: ./.github/actions/rust/post-merge + with: + package: iggy-cli + version: ${{ steps.versions.outputs.cli_version }} + dry_run: ${{ inputs.dry_run }} + + - name: Set final status output + id: final-status + if: always() + run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT" + + publish: + name: ${{ matrix.name }} + needs: [validate, plan, check-tags, build-python-wheels, publish-rust-crates] + if: | + always() && + needs.validate.outputs.has_targets == 'true' && + fromJson(needs.plan.outputs.non_rust_targets).include[0].key != 'noop' && + (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') && + (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan.outputs.non_rust_targets) }} + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} + DRY_RUN: ${{ inputs.dry_run }} + outputs: + status: ${{ steps.status.outputs.status }} + version: ${{ steps.ver.outputs.version }} + tag: ${{ steps.ver.outputs.tag }} + steps: + - name: Save latest CI and scripts from master + if: inputs.use_latest_ci + run: | + # First checkout master to get latest CI and scripts + git clone --depth 1 --branch master https://github.com/${{ github.repository }}.git /tmp/repo-master + cp -r /tmp/repo-master/scripts /tmp/latest-scripts + cp -r /tmp/repo-master/.github /tmp/latest-github + echo "βœ… Saved latest CI and scripts from master branch" + echo "Scripts:" + ls -la /tmp/latest-scripts/ + echo ".github:" + ls -la /tmp/latest-github/ + + - name: Checkout at commit + uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + fetch-depth: 0 + + - name: Apply latest CI and scripts + if: inputs.use_latest_ci + run: | + echo "πŸ“ Applying latest CI and scripts from master branch..." + cp -r /tmp/latest-scripts/* scripts/ + cp -r /tmp/latest-github/* .github/ + chmod +x scripts/*.sh + echo "βœ… Latest CI and scripts applied" + + - name: Ensure version extractor is executable + run: | + test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh + + - name: Setup Rust toolchain (if needed) + if: matrix.type == 'rust' || matrix.type == 'docker' || matrix.type == 'python' + uses: ./.github/actions/utils/setup-rust-with-cache + with: + cache-targets: false + show-stats: false + + - name: Debug matrix + run: echo '${{ toJson(matrix) }}' + + - name: Extract version & tag + id: ver + shell: bash + run: | + set -euo pipefail + GO_FLAG="" + if [ "${{ matrix.key }}" = "sdk-go" ] && [ -n "${{ needs.plan.outputs.go_sdk_version }}" ]; then + GO_FLAG="--go-sdk-version ${{ needs.plan.outputs.go_sdk_version }}" + fi + VERSION=$(scripts/extract-version.sh "${{ matrix.key }}" $GO_FLAG) + # If a tag pattern exists for this component, ask the script for a tag as well + if [ -n "${{ matrix.tag_pattern }}" ] && [ "${{ matrix.tag_pattern }}" != "null" ]; then + TAG=$(scripts/extract-version.sh "${{ matrix.key }}" $GO_FLAG --tag) + else + TAG="" + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "βœ… Resolved ${{ matrix.key }} -> version=$VERSION tag=${TAG:-}" + + # ───────────────────────────────────────── + # Docker Publishing + # ───────────────────────────────────────── + - name: Publish Docker image + if: matrix.type == 'docker' + uses: ./.github/actions/utils/docker-buildx + with: + task: publish + component: ${{ matrix.key }} + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + + # ───────────────────────────────────────── + # Python SDK Publishing + # ───────────────────────────────────────── + - name: Publish Python SDK + if: matrix.type == 'python' + uses: ./.github/actions/python-maturin/post-merge + with: + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + wheels_artifact: python-wheels-all + wheels_path: dist + + # ───────────────────────────────────────── + # Node SDK Publishing + # ───────────────────────────────────────── + - name: Publish Node SDK + if: matrix.type == 'node' + uses: ./.github/actions/node-npm/post-merge + with: + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + + # ───────────────────────────────────────── + # Java SDK Publishing + # ───────────────────────────────────────── + - name: Publish Java SDK + if: matrix.type == 'java' + uses: ./.github/actions/java-gradle/post-merge + with: + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + + # ───────────────────────────────────────── + # C# SDK Publishing + # ───────────────────────────────────────── + - name: Publish C# SDK + if: matrix.type == 'csharp' + uses: ./.github/actions/csharp-dotnet/post-merge + with: + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + + # ───────────────────────────────────────── + # Go Module (Tag-only) + # ───────────────────────────────────────── + - name: Prepare Go tag + if: matrix.type == 'go' + uses: ./.github/actions/go/post-merge + with: + version: ${{ steps.ver.outputs.version }} + dry_run: ${{ inputs.dry_run }} + + - name: Set status output + id: status + if: always() + run: echo "status=${{ job.status }}" >> "$GITHUB_OUTPUT" + + create-tags: + name: Create Git tags + needs: [validate, plan, check-tags, build-python-wheels, publish-rust-crates, publish] + if: | + always() && + needs.validate.outputs.has_targets == 'true' && + inputs.dry_run == false && + (needs.publish.result == 'success' || needs.publish.result == 'skipped') && + (needs.publish-rust-crates.result == 'success' || needs.publish-rust-crates.result == 'skipped') && + (needs.build-python-wheels.result == 'success' || needs.build-python-wheels.result == 'skipped') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Ensure version extractor is executable + run: | + test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh + + - name: Create and push tags (for tagged components) + shell: bash + run: | + set -euo pipefail + TARGETS_JSON='${{ needs.plan.outputs.targets }}' + GO_SDK_VERSION='${{ needs.plan.outputs.go_sdk_version }}' + + echo "$TARGETS_JSON" | jq -r '.include[] | select(.key!="noop") | @base64' | while read -r row; do + _jq() { echo "$row" | base64 -d | jq -r "$1"; } + + KEY=$(_jq '.key') + NAME=$(_jq '.name') + TAG_PATTERN=$(_jq '.tag_pattern') + + # Only components that define tag_pattern will be tagged + if [ -z "$TAG_PATTERN" ] || [ "$TAG_PATTERN" = "null" ]; then + continue + fi + + GO_FLAG="" + if [ "$KEY" = "sdk-go" ] && [ -n "$GO_SDK_VERSION" ]; then + GO_FLAG="--go-sdk-version $GO_SDK_VERSION" + fi + + TAG=$(scripts/extract-version.sh "$KEY" $GO_FLAG --tag) + + echo "Creating tag: $TAG for $NAME" + + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo " ⚠️ Tag $TAG already exists, skipping" + continue + fi + + git tag -a "$TAG" "${{ needs.validate.outputs.commit }}" \ + -m "Release $NAME ($TAG) + Component: $NAME + Tag: $TAG + Commit: ${{ needs.validate.outputs.commit }} + Released by: GitHub Actions + Date: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" + + git push origin "$TAG" + echo " βœ… Created and pushed tag: $TAG" + done + + summary: + name: Publish Summary + needs: [validate, plan, check-tags, build-python-wheels, publish-rust-crates, publish, create-tags] + if: always() && needs.validate.outputs.has_targets == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.validate.outputs.commit }} + + - name: Ensure version extractor is executable + run: | + test -x scripts/extract-version.sh || chmod +x scripts/extract-version.sh + + - name: Generate summary + run: | + { + echo "# πŸ“¦ Publish Summary" + echo + echo "## Configuration" + echo + echo "| Setting | Value |" + echo "|---------|-------|" + echo "| **Commit** | \`${{ needs.validate.outputs.commit }}\` |" + echo "| **Dry run** | \`${{ inputs.dry_run }}\` |" + echo "| **Total components** | ${{ needs.plan.outputs.count }} |" + echo + + # Extract version information for all requested components + echo "## Component Versions" + echo + echo "| Component | Version | Tag | Registry | Status |" + echo "|-----------|---------|-----|----------|--------|" + + # Parse the targets from plan job + TARGETS_JSON='${{ needs.plan.outputs.targets }}' + GO_SDK_VERSION='${{ needs.plan.outputs.go_sdk_version }}' + + echo "$TARGETS_JSON" | jq -r '.include[] | select(.key!="noop") | @base64' | while read -r row; do + _jq() { echo "$row" | base64 -d | jq -r "$1"; } + + KEY=$(_jq '.key') + NAME=$(_jq '.name') + REGISTRY=$(_jq '.registry') + TAG_PATTERN=$(_jq '.tag_pattern') + + # Extract version using the script + GO_FLAG="" + if [ "$KEY" = "sdk-go" ] && [ -n "$GO_SDK_VERSION" ]; then + GO_FLAG="--go-sdk-version $GO_SDK_VERSION" + fi + + VERSION=$(scripts/extract-version.sh "$KEY" $GO_FLAG 2>/dev/null || echo "N/A") + + # Get tag if pattern exists + TAG="" + if [ -n "$TAG_PATTERN" ] && [ "$TAG_PATTERN" != "null" ]; then + TAG=$(scripts/extract-version.sh "$KEY" $GO_FLAG --tag 2>/dev/null || echo "N/A") + else + TAG="N/A" + fi + + # Determine status emoji based on dry run + if [ "${{ inputs.dry_run }}" = "true" ]; then + STATUS="πŸ” Dry run" + else + STATUS="βœ… Published" + fi + + # Format registry display + case "$REGISTRY" in + crates) REGISTRY_DISPLAY="crates.io" ;; + dockerhub) REGISTRY_DISPLAY="Docker Hub" ;; + pypi) REGISTRY_DISPLAY="PyPI" ;; + npm) REGISTRY_DISPLAY="npm" ;; + maven) REGISTRY_DISPLAY="Maven" ;; + nuget) REGISTRY_DISPLAY="NuGet" ;; + none) REGISTRY_DISPLAY="Tag only" ;; + *) REGISTRY_DISPLAY="$REGISTRY" ;; + esac + + echo "| $NAME | \`$VERSION\` | \`$TAG\` | $REGISTRY_DISPLAY | $STATUS |" + done + + echo + + if [ -n "${{ inputs.publish_crates }}" ]; then + echo "### πŸ¦€ Rust Crates Requested" + echo '```' + echo "${{ inputs.publish_crates }}" + echo '```' + fi + if [ -n "${{ inputs.publish_dockerhub }}" ]; then + echo "### 🐳 Docker Images Requested" + echo '```' + echo "${{ inputs.publish_dockerhub }}" + echo '```' + fi + if [ -n "${{ inputs.publish_other }}" ]; then + echo "### πŸ“¦ Other SDKs Requested" + echo '```' + echo "${{ inputs.publish_other }}" + echo '```' + fi + echo + echo "## Results" + echo + + # Python wheels building status + if [ "${{ needs.plan.outputs.has_python }}" = "true" ]; then + echo "### Python Wheels Building" + case "${{ needs.build-python-wheels.result }}" in + success) echo "βœ… **Python wheels built successfully for all platforms**" ;; + failure) echo "❌ **Python wheel building failed**" ;; + skipped) echo "⏭️ **Python wheel building was skipped**" ;; + esac + echo + fi + + # Rust crates publishing status + if [ -n "${{ inputs.publish_crates }}" ]; then + echo "### Rust Crates Publishing (Sequential)" + case "${{ needs.publish-rust-crates.result }}" in + success) echo "βœ… **Rust crates published successfully in dependency order**" ;; + failure) echo "❌ **Rust crates publishing failed - check logs for details**" ;; + skipped) echo "⏭️ **Rust crates publishing was skipped**" ;; + esac + echo + fi + + # Other publishing status + echo "### Other Publishing" + case "${{ needs.publish.result }}" in + success) echo "βœ… **Publishing completed successfully**" ;; + failure) echo "❌ **Publishing failed - check logs for details**" ;; + cancelled) echo "🚫 **Publishing was cancelled**" ;; + *) echo "⏭️ **Publishing was skipped**" ;; + esac + if [ "${{ inputs.dry_run }}" = "true" ]; then + echo + echo "**ℹ️ This was a dry run - no actual publishing occurred**" + else + case "${{ needs.create-tags.result }}" in + success) echo "βœ… **Git tags created successfully**" ;; + failure) echo "⚠️ **Tag creation had issues**" ;; + skipped) echo "⏭️ **Tag creation was skipped (dry run or publish failed)**" ;; + esac + fi + echo + echo "---" + echo "*Workflow completed at $(date -u +"%Y-%m-%d %H:%M:%S UTC")*" + } >> "$GITHUB_STEP_SUMMARY" + + notify-failure: + name: Notify on failure + needs: [validate, plan, build-python-wheels, publish-rust-crates, publish, create-tags, summary] + if: failure() && inputs.dry_run == false + runs-on: ubuntu-latest + steps: + - name: Notify failure + run: | + echo "❌ Publishing workflow failed!" + echo "Check the workflow run for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/publish_bench.yml b/.github/workflows/publish_bench.yml deleted file mode 100644 index 3d7bae08fa..0000000000 --- a/.github/workflows/publish_bench.yml +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_bench -on: - push: - tags: - - "iggy-bench-*" - -env: - GITHUB_TOKEN: ${{ github.token }} - CARGO_TERM_COLOR: always - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/iggy-bench-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^iggy-bench-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (iggy-bench-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy-bench version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p iggy-bench | cut -d@ -f2) - echo "iggy_bench_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "iggy-bench-${{ steps.extract_version.outputs.iggy_bench_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - iggy_bench_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish bench on crates.io - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: publish - run: | - cargo login "${{ secrets.CARGO_REGISTRY_TOKEN }}" - cargo publish -p iggy-bench - - finalize_bench: - runs-on: ubuntu-latest - needs: - - publish - if: always() - steps: - - uses: actions/checkout@v4 - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "publish to crates.io" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_cli.yml b/.github/workflows/publish_cli.yml deleted file mode 100644 index 8e23d5b290..0000000000 --- a/.github/workflows/publish_cli.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_cli -on: - push: - tags: - - 'iggy-cli-*' - -env: - GITHUB_TOKEN: ${{ github.token }} - CARGO_TERM_COLOR: always - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/iggy-cli-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^iggy-cli-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (iggy-cli-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy-cli version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p iggy-cli | cut -d@ -f2) - echo "iggy_cli_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "iggy-cli-${{ steps.extract_version.outputs.iggy_cli_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - iggy_cli_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish CLI on crates.io - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: publish - run: | - cargo login "${{ secrets.CARGO_REGISTRY_TOKEN }}" - cargo publish -p iggy-cli - - github_release: - uses: ./.github/workflows/release_cli.yml - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - with: - tag_name: "iggy-cli-${{ needs.tag.outputs.iggy_cli_version }}" - - finalize_cli: - runs-on: ubuntu-latest - needs: - - publish - - github_release - if: always() - steps: - - uses: actions/checkout@v4 - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "publish to crates.io" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml deleted file mode 100644 index d99e52f4c7..0000000000 --- a/.github/workflows/publish_connectors.yml +++ /dev/null @@ -1,285 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_connectors - -on: - push: - tags: - - "connectors-*" - -env: - DOCKERHUB_REGISTRY_NAME: apache/iggy-connect - CRATE_NAME: iggy-connectors - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/connectors-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^connectors-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (connectors-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy-connectors version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p iggy-connectors | cut -d# -f2 | cut -d: -f2) - echo "connectors_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "connectors-${{ steps.extract_version.outputs.connectors_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - connectors_version: ${{ steps.extract_version.outputs.connectors_version }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - release_and_publish: - name: build release ${{ matrix.platform.os_name }} - needs: tag - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - os_name: Linux-x86_64-musl - os: ubuntu-latest - target: x86_64-unknown-linux-musl - bin: - - iggy-connectors - name: iggy-connectors-Linux-x86_64-musl.tar.gz - cargo_command: cargo - docker_arch: linux/amd64 - cross: false - - - os_name: Linux-aarch64-musl - os: ubuntu-latest - target: aarch64-unknown-linux-musl - bin: - - iggy-connectors - name: iggy-connectors-Linux-aarch64-musl.tar.gz - docker_arch: linux/arm64/v8 - cross: true - - toolchain: - - stable - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "publish-connectors-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: contains(matrix.platform.name, 'musl') - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build iggy-connectors ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} --bin iggy-connectors - - - name: Build iggy-cli ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --no-default-features --target ${{ matrix.platform.target }} --bin iggy - - - name: Prepare artifacts - run: | - mkdir -p release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy-connectors release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy release_artifacts/ - - - name: Print message - run: echo "::notice ::Published ${{ needs.tag.outputs.connectors_version }} release artifacts on GitHub" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: "arm64,arm" - if: ${{ matrix.platform.cross }} - - - name: Set up Docker - uses: docker/setup-docker-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./Dockerfile.connectors.ci - platforms: ${{ matrix.platform.docker_arch }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.DOCKERHUB_REGISTRY_NAME }},push-by-digest=true,name-canonical=true,push=true - build-args: | - IGGY_CONNECTORS_PATH=target/${{ matrix.platform.target }}/release/iggy-connectors - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform.os_name }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge_docker_manifest: - runs-on: ubuntu-latest - needs: - - release_and_publish - - tag - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: "digests-*" - merge-multiple: true - path: /tmp/digests - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta (tag was created) - if: ${{ needs.tag.outputs.tag_created == 'true' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - ${{ needs.tag.outputs.connectors_version }} - latest - - - name: Docker meta (tag was not created) - if: ${{ needs.tag.outputs.tag_created == '' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - edge - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKERHUB_REGISTRY_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKERHUB_REGISTRY_NAME }}:latest - - github_release: - uses: ./.github/workflows/release_connectors.yml - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - with: - tag_name: "connectors-${{ needs.tag.outputs.connectors_version }}" - - finalize_publish_connectors: - runs-on: ubuntu-latest - needs: - - release_and_publish - - merge_docker_manifest - - github_release - if: always() - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name != 'workflow_dispatch' }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "build and release to dockerhub" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_csharp_sdk.yml b/.github/workflows/publish_csharp_sdk.yml deleted file mode 100644 index bc41a0cc06..0000000000 --- a/.github/workflows/publish_csharp_sdk.yml +++ /dev/null @@ -1,112 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_csharp_sdk -on: - push: - tags: - - 'csharp-sdk-*' - -env: - GITHUB_TOKEN: ${{ github.token }} - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/csharp-sdk-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^csharp-sdk-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (csharp-sdk-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract csharp sdk version from csproj - id: extract_version - working-directory: foreign/csharp/Iggy_SDK - run: | - # Check if Version property exists in csproj - if grep -q "" Iggy_SDK.csproj; then - version=$(grep -oP '(?<=)[^<]*' Iggy_SDK.csproj) - echo "csharp_sdk_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version csharp sdk csproj version is $version" - else - echo "::error ::No Version property found in csproj. Please add X.Y.Z to Iggy_SDK.csproj" - exit 1 - fi - - - name: Check if version from csproj is the same as the tag - id: check_git_tag - run: | - if [[ "csharp-sdk-${{ steps.extract_version.outputs.csharp_sdk_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in csproj" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from csproj" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - csharp_sdk_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish csharp SDK - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup .NET 8 - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '8.0.x' - - name: Restore dependencies - working-directory: foreign/csharp/Iggy_SDK - run: dotnet restore - - name: Pack nuget - working-directory: foreign/csharp/Iggy_SDK - run: | - dotnet pack -c release --no-restore -o . - - name: Publish nuget - working-directory: foreign/csharp/Iggy_SDK - run: dotnet nuget push *.nupkg -k "${{ secrets.NUGET_API_KEY }}" -s https://api.nuget.org/v3/index.json - diff --git a/.github/workflows/publish_java_sdk.yml b/.github/workflows/publish_java_sdk.yml deleted file mode 100644 index 450fb01c36..0000000000 --- a/.github/workflows/publish_java_sdk.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_java_sdk -on: - push: - tags: - - "java-sdk-*" - -env: - GITHUB_TOKEN: ${{ github.token }} - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/java-sdk-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^java-sdk-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (java-sdk-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract java-sdk version from build.gradle.kts - id: extract_version - run: | - version=$(foreign/java/gradlew -p foreign/java/java-sdk properties -q | grep "version:" | cut -d: -f2 | tr -d ' ') - echo "java_sdk_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from gradle build file is $version" - - - name: Check if version from build.gradle is the same as the tag - id: check_git_tag - run: | - if [[ "java-sdk-${{ steps.extract_version.outputs.java_sdk_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in build.gradle" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not matche the version from build.gradle" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - java_sdk_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish Java SDK to Maven Nexus - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Java - uses: actions/setup-java@v4 - with: - java-version: "17" - distribution: "temurin" - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 - - - name: Build - run: foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest - - - name: Run tests - run: foreign/java/dev-support/checks/build.sh test - - - name: Publish - # To publish we use NEXUS_USER and NEXUS_PASSWORD as credentials - env: - NEXUS_USER: ${{ secrets.NEXUS_USER }} - NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }} - run: foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest publish diff --git a/.github/workflows/publish_java_sdk_snapshots.yml b/.github/workflows/publish_java_sdk_snapshots.yml deleted file mode 100644 index 4e86a89b7d..0000000000 --- a/.github/workflows/publish_java_sdk_snapshots.yml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish java-sdk SNAPSHOTs - -on: - schedule: - - cron: "5 0 * * *" - -jobs: - publish: - if: github.repository == 'apache/iggy' - name: Publish Snapshots - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Java - uses: actions/setup-java@v4 - with: - java-version: "17" - distribution: "temurin" - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 - - - name: Build - run: foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest -x sign - - - name: Run tests - run: foreign/java/dev-support/checks/build.sh test - - - id: extract_version - name: Extract version - shell: bash - if: ${{ success() }} - run: | - VERSION=$(foreign/java/gradlew -p foreign/java/java-sdk properties -q | grep "version:") - if [[ "$VERSION" == *"SNAPSHOT"* ]]; then - echo "snapshot=SNAPSHOT" >> $GITHUB_OUTPUT - fi - - - name: Publish Snapshot - # To publish we use NEXUS_USER and NEXUS_PASSWORD as credentials - env: - NEXUS_USER: ${{ secrets.NEXUS_USER }} - NEXUS_PASSWORD: ${{ secrets.NEXUS_PW }} - if: steps.extract_version.outputs.snapshot == 'SNAPSHOT' - run: foreign/java/dev-support/checks/build.sh build -x test -x checkstyleMain -x checkstyleTest -x sign publish diff --git a/.github/workflows/publish_mcp.yml b/.github/workflows/publish_mcp.yml deleted file mode 100644 index 6a6c72dab8..0000000000 --- a/.github/workflows/publish_mcp.yml +++ /dev/null @@ -1,285 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_mcp - -on: - push: - tags: - - "mcp-*" - -env: - DOCKERHUB_REGISTRY_NAME: apache/iggy-mcp - CRATE_NAME: iggy-mcp - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/mcp-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^mcp-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (mcp-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy-mcp version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p iggy-mcp | cut -d# -f2 | cut -d: -f2) - echo "mcp_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "mcp-${{ steps.extract_version.outputs.mcp_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - mcp_version: ${{ steps.extract_version.outputs.mcp_version }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - release_and_publish: - name: build release ${{ matrix.platform.os_name }} - needs: tag - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - os_name: Linux-x86_64-musl - os: ubuntu-latest - target: x86_64-unknown-linux-musl - bin: - - iggy-mcp - name: iggy-mcp-Linux-x86_64-musl.tar.gz - cargo_command: cargo - docker_arch: linux/amd64 - cross: false - - - os_name: Linux-aarch64-musl - os: ubuntu-latest - target: aarch64-unknown-linux-musl - bin: - - iggy-mcp - name: iggy-mcp-Linux-aarch64-musl.tar.gz - docker_arch: linux/arm64/v8 - cross: true - - toolchain: - - stable - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "publish-mcp-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: contains(matrix.platform.name, 'musl') - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build iggy-mcp ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} --bin iggy-mcp - - - name: Build iggy-cli ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --no-default-features --target ${{ matrix.platform.target }} --bin iggy - - - name: Prepare artifacts - run: | - mkdir -p release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy-mcp release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy release_artifacts/ - - - name: Print message - run: echo "::notice ::Published ${{ needs.tag.outputs.mcp }} release artifacts on GitHub" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: "arm64,arm" - if: ${{ matrix.platform.cross }} - - - name: Set up Docker - uses: docker/setup-docker-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./Dockerfile.mcp.ci - platforms: ${{ matrix.platform.docker_arch }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.DOCKERHUB_REGISTRY_NAME }},push-by-digest=true,name-canonical=true,push=true - build-args: | - IGGY_MCP_PATH=target/${{ matrix.platform.target }}/release/iggy-mcp - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform.os_name }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge_docker_manifest: - runs-on: ubuntu-latest - needs: - - release_and_publish - - tag - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: "digests-*" - merge-multiple: true - path: /tmp/digests - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta (tag was created) - if: ${{ needs.tag.outputs.tag_created == 'true' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - ${{ needs.tag.outputs.mcp_version }} - latest - - - name: Docker meta (tag was not created) - if: ${{ needs.tag.outputs.tag_created == '' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - edge - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKERHUB_REGISTRY_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKERHUB_REGISTRY_NAME }}:latest - - github_release: - uses: ./.github/workflows/release_mcp.yml - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - with: - tag_name: "mcp-${{ needs.tag.outputs.mcp_version }}" - - finalize_publish_mcp: - runs-on: ubuntu-latest - needs: - - release_and_publish - - merge_docker_manifest - - github_release - if: always() - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name != 'workflow_dispatch' }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "build and release to dockerhub" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_node_sdk.yml b/.github/workflows/publish_node_sdk.yml deleted file mode 100644 index 5a91713956..0000000000 --- a/.github/workflows/publish_node_sdk.yml +++ /dev/null @@ -1,129 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_node_sdk -on: - push: - tags: - - 'node-sdk-*' - -env: - GITHUB_TOKEN: ${{ github.token }} - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/node-sdk-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^node-sdk-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (node-sdk-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract node sdk version from package.json - id: extract_version - run: | - version=`grep -n '"version": ' package.json | sed -E 's/^.* "([^"]*)",/\1/' | tr -d '\n'` - echo "node_sdk_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version node sdk package.json version is $version" - - - name: Check if version from package.json is the same as the tag - id: check_git_tag - run: | - if [[ "node-sdk-${{ steps.extract_version.outputs.node_sdk_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in package.json" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not matche the version from package.json" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - node_sdk_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish SDK on npmjs - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - # Setup .npmrc file to publish to npm - - uses: actions/setup-node@v4 - with: - cache-dependency-path: foreign/node/package-lock.json - node-version: 22 - cache: "npm" - registry-url: 'https://registry.npmjs.org' - - run: npm ci - - run: npm run build - - run: npm publish --provenance --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - # github_release: - # uses: ./.github/workflows/release_sdk.yml - # needs: tag - # if: ${{ needs.tag.outputs.tag_created == 'true' }} - # with: - # tag_name: "${{ needs.tag.outputs.node_sdk_version }}" - - # finalize_sdk: - # runs-on: ubuntu-latest - # needs: - # - publish - # - github_release - # if: always() - # steps: - # - uses: actions/checkout@v4 - # - name: Everything is fine - # if: ${{ !(contains(needs.*.result, 'failure')) }} - # run: exit 0 - # - name: Something went wrong - # if: ${{ contains(needs.*.result, 'failure') }} - # uses: JasonEtco/create-an-issue@v2 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # GITHUB_BOT_CONTEXT_STRING: "publish to crates.io" - # with: - # filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_rust_sdk.yml b/.github/workflows/publish_rust_sdk.yml deleted file mode 100644 index 28efbccd7a..0000000000 --- a/.github/workflows/publish_rust_sdk.yml +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_rust_sdk -on: - push: - tags: - - "iggy-*" - -env: - GITHUB_TOKEN: ${{ github.token }} - CARGO_TERM_COLOR: always - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/iggy-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^iggy-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (iggy-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p iggy | cut -d@ -f2) - echo "iggy_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "iggy-${{ steps.extract_version.outputs.iggy_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not matche the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - iggy_version: ${{ steps.extract_tag.outputs.tag_name }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - publish: - name: Publish SDK on crates.io - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: publish - run: | - cargo login "${{ secrets.CARGO_REGISTRY_TOKEN }}" - cargo publish -p iggy - - github_release: - uses: ./.github/workflows/release_sdk.yml - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - with: - tag_name: "${{ needs.tag.outputs.iggy_version }}" - - finalize_sdk: - runs-on: ubuntu-latest - needs: - - publish - - github_release - if: always() - steps: - - uses: actions/checkout@v4 - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "publish to crates.io" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/publish_server.yml b/.github/workflows/publish_server.yml deleted file mode 100644 index 0cf20b4793..0000000000 --- a/.github/workflows/publish_server.yml +++ /dev/null @@ -1,288 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: publish_server - -on: - push: - tags: - - "server-*" - -env: - DOCKERHUB_REGISTRY_NAME: apache/iggy - CRATE_NAME: iggy - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - validate: - if: startsWith(github.ref, 'refs/tags/server-') - runs-on: ubuntu-latest - steps: - - name: Extract tag name - id: extract - run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - - name: Validate tag format - run: | - TAG=${TAG} - if [[ ! "$TAG" =~ ^server-([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$ ]]; then - echo "Tag $TAG does not match strict semver format (server-X.Y.Z where 0 <= X,Y,Z <= 999)" - exit 1 - fi - echo "Valid tag: $TAG" - - tag: - needs: validate - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Extract tag name - id: extract_tag - run: | - tag=${GITHUB_REF#refs/tags/} - echo "tag_name=$tag" >> "$GITHUB_OUTPUT" - echo "::notice ::Tag that triggered the workflow: $tag" - - - name: Extract iggy-server version from Cargo.toml - id: extract_version - run: | - version=$(cargo pkgid -p server | cut -d# -f2 | cut -d: -f2) - echo "server_version=$version" >> "$GITHUB_OUTPUT" - echo "::notice ::Version from Cargo.toml $version" - - - name: Check if version from Cargo.toml is the same as the tag - id: check_git_tag - run: | - if [[ "server-${{ steps.extract_version.outputs.server_version }}" == "${{ steps.extract_tag.outputs.tag_name }}" ]]; - then - echo "::notice ::Tag ${{ steps.extract_tag.outputs.tag_name }} matches the version in Cargo.toml" - echo "tag_matches=true" >> "$GITHUB_OUTPUT" - else - echo "::warning ::Tag ${{ steps.extract_tag.outputs.tag_name }} does not match the version from Cargo.toml" - echo "tag_matches=false" >> "$GITHUB_OUTPUT" - fi - - outputs: - server_version: ${{ steps.extract_version.outputs.server_version }} - tag_created: ${{ steps.check_git_tag.outputs.tag_matches }} - - release_and_publish: - name: build release ${{ matrix.platform.os_name }} - needs: tag - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - os_name: Linux-x86_64-musl - os: ubuntu-latest - target: x86_64-unknown-linux-musl - bin: - - iggy-server - - iggy - name: iggy-Linux-x86_64-musl.tar.gz - cargo_command: cargo - docker_arch: linux/amd64 - cross: false - - - os_name: Linux-aarch64-musl - os: ubuntu-latest - target: aarch64-unknown-linux-musl - bin: - - iggy-server - - iggy - name: iggy-Linux-aarch64-musl.tar.gz - docker_arch: linux/arm64/v8 - cross: true - - toolchain: - - stable - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "publish-server-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: contains(matrix.platform.name, 'musl') - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build iggy-server ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} --bin iggy-server - - - name: Build iggy-cli ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --no-default-features --target ${{ matrix.platform.target }} --bin iggy - - - name: Prepare artifacts - run: | - mkdir -p release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy-server release_artifacts/ - cp target/${{ matrix.platform.target }}/release/iggy release_artifacts/ - - - name: Print message - run: echo "::notice ::Published ${{ needs.tag.outputs.server_version }} release artifacts on GitHub" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: "arm64,arm" - if: ${{ matrix.platform.cross }} - - - name: Set up Docker - uses: docker/setup-docker-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: ./ - file: ./Dockerfile.ci - platforms: ${{ matrix.platform.docker_arch }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.DOCKERHUB_REGISTRY_NAME }},push-by-digest=true,name-canonical=true,push=true - build-args: | - IGGY_CMD_PATH=target/${{ matrix.platform.target }}/release/iggy - IGGY_SERVER_PATH=target/${{ matrix.platform.target }}/release/iggy-server - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.platform.os_name }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge_docker_manifest: - runs-on: ubuntu-latest - needs: - - release_and_publish - - tag - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: "digests-*" - merge-multiple: true - path: /tmp/digests - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta (tag was created) - if: ${{ needs.tag.outputs.tag_created == 'true' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - ${{ needs.tag.outputs.server_version }} - latest - - - name: Docker meta (tag was not created) - if: ${{ needs.tag.outputs.tag_created == '' }} - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKERHUB_REGISTRY_NAME }} - tags: | - edge - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKERHUB_REGISTRY_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKERHUB_REGISTRY_NAME }}:latest - - github_release: - uses: ./.github/workflows/release_server.yml - needs: tag - if: ${{ needs.tag.outputs.tag_created == 'true' }} - with: - tag_name: "server-${{ needs.tag.outputs.server_version }}" - - finalize_publish_server: - runs-on: ubuntu-latest - needs: - - release_and_publish - - merge_docker_manifest - - github_release - if: always() - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name != 'workflow_dispatch' }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "build and release to dockerhub" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/release_bench.yml b/.github/workflows/release_bench.yml deleted file mode 100644 index c20bf2ad29..0000000000 --- a/.github/workflows/release_bench.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_bench - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - build_bench: - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - target: x86_64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-bench - file: iggy-bench-x86_64-unknown-linux-musl.tgz - - target: aarch64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-bench - file: iggy-bench-aarch64-unknown-linux-musl.tgz - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - executable: iggy-bench - file: iggy-bench-x86_64-unknown-linux-gnu.tgz - - target: x86_64-pc-windows-msvc - os: windows-latest - executable: iggy-bench.exe - file: iggy-bench-x86_64-pc-windows-msvc.zip - - target: aarch64-pc-windows-msvc - os: windows-latest - executable: iggy-bench.exe - file: iggy-bench-aarch64-pc-windows-msvc.zip - - target: x86_64-apple-darwin - os: macos-latest - executable: iggy-bench - file: iggy-bench-x86_64-apple-darwin.zip - - target: aarch64-apple-darwin - os: macos-latest - executable: iggy-bench - file: iggy-bench-aarch64-apple-darwin.zip - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "release-bench-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: ${{ matrix.platform.target == 'x86_64-unknown-linux-musl' }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} -p iggy-bench - - - name: Collect ${{ matrix.platform.target }} executable - run: | - cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.executable }} . - - - name: Create ${{ matrix.platform.file }} artifact - run: | - tar cvfz ${{ matrix.platform.file }} ${{ matrix.platform.executable }} - if: ${{ matrix.platform.os == 'ubuntu-latest' }} - - - name: Create ${{ matrix.platform.file }} artifact - uses: vimtor/action-zip@v1 - with: - files: ${{ matrix.platform.executable }} - dest: ${{ matrix.platform.file }} - if: ${{ matrix.platform.os == 'windows-latest' || matrix.platform.os == 'macos-latest' }} - - - name: Upload ${{ matrix.platform.file }} artifact - uses: actions/upload-artifact@v4 - with: - name: artifacts-${{ matrix.platform.target }} - path: ${{ matrix.platform.file }} - - - name: Print message - run: echo "::notice ::Created binary for ${{ matrix.platform.target }}" - - outputs: - version: ${{ needs.tag.outputs.version }} diff --git a/.github/workflows/release_cli.yml b/.github/workflows/release_cli.yml deleted file mode 100644 index 4c1a74894c..0000000000 --- a/.github/workflows/release_cli.yml +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_cli - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - build_cli: - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - target: x86_64-unknown-linux-musl - os: ubuntu-latest - executable: iggy - file: iggy-cli-x86_64-unknown-linux-musl.tgz - - target: aarch64-unknown-linux-musl - os: ubuntu-latest - executable: iggy - file: iggy-cli-aarch64-unknown-linux-musl.tgz - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - executable: iggy - file: iggy-cli-x86_64-unknown-linux-gnu.tgz - - target: x86_64-pc-windows-msvc - os: windows-latest - executable: iggy.exe - file: iggy-cli-x86_64-pc-windows-msvc.zip - - target: aarch64-pc-windows-msvc - os: windows-latest - executable: iggy.exe - file: iggy-cli-aarch64-pc-windows-msvc.zip - - target: x86_64-apple-darwin - os: macos-latest - executable: iggy - file: iggy-cli-x86_64-apple-darwin.zip - - target: aarch64-apple-darwin - os: macos-latest - executable: iggy - file: iggy-cli-aarch64-apple-darwin.zip - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "release-cli-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: ${{ matrix.platform.target == 'x86_64-unknown-linux-musl' }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} -p iggy - - - name: Collect ${{ matrix.platform.target }} executable - run: | - cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.executable }} . - - - name: Create ${{ matrix.platform.file }} artifact - run: | - tar cvfz ${{ matrix.platform.file }} ${{ matrix.platform.executable }} - if: ${{ matrix.platform.os == 'ubuntu-latest' }} - - - name: Create ${{ matrix.platform.file }} artifact - uses: vimtor/action-zip@v1 - with: - files: ${{ matrix.platform.executable }} - dest: ${{ matrix.platform.file }} - if: ${{ matrix.platform.os == 'windows-latest' || matrix.platform.os == 'macos-latest' }} - - - name: Upload ${{ matrix.platform.file }} artifact - uses: actions/upload-artifact@v4 - with: - name: artifacts-${{ matrix.platform.target }} - path: ${{ matrix.platform.file }} - - - name: Print message - run: echo "::notice ::Created binary for ${{ matrix.platform.target }}" - - outputs: - version: ${{ needs.tag.outputs.version }} - - release_cli: - name: Create iggy-cli release - runs-on: ubuntu-latest - needs: build_cli - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Download all artifacts - uses: actions/download-artifact@v4 - - - name: List files - run: find - - # Action disallowed - # - name: Create Changelog - # uses: orhun/git-cliff-action@v4 - # id: changelog - # with: - # config: cliff.toml - # args: -vv --latest - # env: - # OUTPUT: CHANGELOG.md - # GITHUB_REPO: ${{ github.repository }} - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - body: ${{ steps.changelog.outputs.content }} - files: | - artifacts-*/*.tgz - artifacts-*/*.zip - CHANGELOG.md - tag_name: ${{ inputs.tag_name }} - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release_connectors.yml b/.github/workflows/release_connectors.yml deleted file mode 100644 index 7e7f01c894..0000000000 --- a/.github/workflows/release_connectors.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_connectors - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - build_connectors: - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - target: x86_64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-connectors - file: iggy-connectors-x86_64-unknown-linux-musl.tgz - - target: aarch64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-connectors - file: iggy-connectors-aarch64-unknown-linux-musl.tgz - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - executable: iggy-connectors - file: iggy-connectors-x86_64-unknown-linux-gnu.tgz - - target: x86_64-pc-windows-msvc - os: windows-latest - executable: iggy-connectors.exe - file: iggy-connectors-x86_64-pc-windows-msvc.zip - - target: aarch64-pc-windows-msvc - os: windows-latest - executable: iggy-connectors.exe - file: iggy-connectors-aarch64-pc-windows-msvc.zip - - target: x86_64-apple-darwin - os: macos-latest - executable: iggy-connectors - file: iggy-connectors-x86_64-apple-darwin.zip - - target: aarch64-apple-darwin - os: macos-latest - executable: iggy-connectors - file: iggy-connectors-aarch64-apple-darwin.zip - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "release-connectors-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: ${{ matrix.platform.target == 'x86_64-unknown-linux-musl' }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} -p iggy-connectors - - - name: Collect ${{ matrix.platform.target }} executable - run: | - cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.executable }} . - - - name: Create ${{ matrix.platform.file }} artifact - run: | - tar cvfz ${{ matrix.platform.file }} ${{ matrix.platform.executable }} - if: ${{ matrix.platform.os == 'ubuntu-latest' }} - - - name: Create ${{ matrix.platform.file }} artifact - uses: vimtor/action-zip@v1 - with: - files: ${{ matrix.platform.executable }} - dest: ${{ matrix.platform.file }} - if: ${{ matrix.platform.os == 'windows-latest' || matrix.platform.os == 'macos-latest' }} - - - name: Upload ${{ matrix.platform.file }} artifact - uses: actions/upload-artifact@v4 - with: - name: artifacts-${{ matrix.platform.target }} - path: ${{ matrix.platform.file }} - - - name: Print message - run: echo "::notice ::Created binary for ${{ matrix.platform.target }}" - - outputs: - version: ${{ needs.tag.outputs.version }} diff --git a/.github/workflows/release_mcp.yml b/.github/workflows/release_mcp.yml deleted file mode 100644 index a1442ae1ce..0000000000 --- a/.github/workflows/release_mcp.yml +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_mcp - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - build_mcp: - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: - - target: x86_64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-mcp - file: iggy-mcp-x86_64-unknown-linux-musl.tgz - - target: aarch64-unknown-linux-musl - os: ubuntu-latest - executable: iggy-mcp - file: iggy-mcp-aarch64-unknown-linux-musl.tgz - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - executable: iggy-mcp - file: iggy-mcp-x86_64-unknown-linux-gnu.tgz - - target: x86_64-pc-windows-msvc - os: windows-latest - executable: iggy-mcp.exe - file: iggy-mcp-x86_64-pc-windows-msvc.zip - - target: aarch64-pc-windows-msvc - os: windows-latest - executable: iggy-mcp.exe - file: iggy-mcp-aarch64-pc-windows-msvc.zip - - target: x86_64-apple-darwin - os: macos-latest - executable: iggy-mcp - file: iggy-mcp-x86_64-apple-darwin.zip - - target: aarch64-apple-darwin - os: macos-latest - executable: iggy-mcp - file: iggy-mcp-aarch64-apple-darwin.zip - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "release-mcp-${{ matrix.platform.target }}" - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - if: ${{ matrix.platform.target == 'x86_64-unknown-linux-musl' }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build ${{ matrix.platform.target }} release binary - run: cross +stable build --verbose --release --target ${{ matrix.platform.target }} -p iggy-mcp - - - name: Collect ${{ matrix.platform.target }} executable - run: | - cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.executable }} . - - - name: Create ${{ matrix.platform.file }} artifact - run: | - tar cvfz ${{ matrix.platform.file }} ${{ matrix.platform.executable }} - if: ${{ matrix.platform.os == 'ubuntu-latest' }} - - - name: Create ${{ matrix.platform.file }} artifact - uses: vimtor/action-zip@v1 - with: - files: ${{ matrix.platform.executable }} - dest: ${{ matrix.platform.file }} - if: ${{ matrix.platform.os == 'windows-latest' || matrix.platform.os == 'macos-latest' }} - - - name: Upload ${{ matrix.platform.file }} artifact - uses: actions/upload-artifact@v4 - with: - name: artifacts-${{ matrix.platform.target }} - path: ${{ matrix.platform.file }} - - - name: Print message - run: echo "::notice ::Created binary for ${{ matrix.platform.target }}" - - outputs: - version: ${{ needs.tag.outputs.version }} diff --git a/.github/workflows/release_sdk.yml b/.github/workflows/release_sdk.yml deleted file mode 100644 index cd2fed6a8b..0000000000 --- a/.github/workflows/release_sdk.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_sdk - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - -jobs: - release_sdk: - name: Release SDK - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Action disallowed - # - name: Create Changelog - # uses: orhun/git-cliff-action@v4 - # id: changelog - # with: - # config: cliff.toml - # args: -vv --latest - # env: - # OUTPUT: CHANGELOG.md - # GITHUB_REPO: ${{ github.repository }} - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - body: ${{ steps.changelog.outputs.content }} - files: | - CHANGELOG.md - tag_name: ${{ inputs.tag_name }} - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - finalize_release: - name: Finalize release - runs-on: ubuntu-latest - needs: - - release_sdk - if: always() - steps: - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Some checks failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 diff --git a/.github/workflows/release_server.yml b/.github/workflows/release_server.yml deleted file mode 100644 index 0b6aff2926..0000000000 --- a/.github/workflows/release_server.yml +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: release_server - -on: - workflow_call: - inputs: - tag_name: - description: "The name of the tag to be released" - required: true - type: string - -env: - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - -jobs: - release_server: - name: Build and release server binary - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install musl-tools on Linux - run: sudo apt-get update --yes && sudo apt-get install --yes musl-tools - - - name: Prepare x86_64-unknown-linux-musl toolchain - run: | - rustup target add x86_64-unknown-linux-musl - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build iggy-server release binary for x86_64-unknown-linux-musl - run: cross +stable build --verbose --target x86_64-unknown-linux-musl --release --bin iggy-server --target-dir target_x86 - - - name: Prepare x86_64-unknown-linux-musl artifacts - run: | - mkdir -p all_artifacts/Linux-x86_64 - cp target_x86/x86_64-unknown-linux-musl/release/iggy-server all_artifacts/Linux-x86_64/ - - - name: Prepare aarch64-unknown-linux-musl toolchain - run: | - rustup target add aarch64-unknown-linux-musl - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Build iggy-server release binary for aarch64-unknown-linux-musl - run: cross +stable build --verbose --target aarch64-unknown-linux-musl --release --bin iggy-server --target-dir target_aarch64 - - - name: Prepare aarch64-unknown-linux-musl artifacts - run: | - mkdir -p all_artifacts/Linux-aarch64 - cp target_aarch64/aarch64-unknown-linux-musl/release/iggy-server all_artifacts/Linux-aarch64/ - - - name: Zip artifacts for each platform - run: | - mkdir zipped_artifacts - for dir in all_artifacts/*; do - if [ -d "$dir" ]; then - zip -r "zipped_artifacts/$(basename $dir).zip" "$dir" - fi - done - - # Action disallowed - # - name: Create Changelog - # uses: orhun/git-cliff-action@v4 - # id: changelog - # with: - # config: cliff.toml - # args: -vv --latest - # env: - # OUTPUT: CHANGELOG.md - # GITHUB_REPO: ${{ github.repository }} - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - body: ${{ steps.changelog.outputs.content }} - files: | - zipped_artifacts/* - CHANGELOG.md - tag_name: ${{ inputs.tag_name }} - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - finalize_release: - name: Finalize release - runs-on: ubuntu-latest - needs: - - release_server - if: always() - steps: - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - - name: Some checks failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml deleted file mode 100644 index 63ad6f3f33..0000000000 --- a/.github/workflows/security.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: security -on: - workflow_dispatch: - schedule: - - cron: '0 8 * * *' # At 8:00 AM UTC, which is 9:00 AM CET - -jobs: - audit: - name: Run cargo audit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: rustsec/audit-check@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - clippy: - name: Run cargo clippy - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - run: | - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install clippy-sarif sarif-fmt - run: cargo install clippy-sarif sarif-fmt - - - name: Run cargo clippy and convert to SARIF - run: - cargo clippy - --all-features --all-targets - --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt - continue-on-error: true - - - name: Upload analysis results to GitHub - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: rust-clippy-results.sarif - wait-for-processing: true diff --git a/.github/workflows/test_daily.yml b/.github/workflows/test_daily.yml deleted file mode 100644 index 9cc4c72a3b..0000000000 --- a/.github/workflows/test_daily.yml +++ /dev/null @@ -1,125 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: test_daily -on: - workflow_dispatch: - schedule: - - cron: '0 8 * * *' # At 8:00 AM UTC, which is 9:00 AM CET - -env: - CRATE_NAME: iggy - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - # Option needed for starting docker under cross to be able to lock memory - # in order to be able to use keyring inside Docker container - CROSS_CONTAINER_OPTS: "--cap-add ipc_lock" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build_and_test: - name: 'build and test ${{ matrix.toolchain }} ${{ matrix.platform.os_name }}' - runs-on: ${{ matrix.platform.os }} - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - platform: - - - os_name: Linux-x86_64-musl - os: ubuntu-latest - target: x86_64-unknown-linux-musl - name: iggy-Linux-x86_64-musl.tar.gz - cargo_command: cargo - profile: release - docker_arch: linux/amd64 - cross: false - - - os_name: Linux-aarch64-musl - os: ubuntu-latest - target: aarch64-unknown-linux-musl - name: iggy-Linux-aarch64-musl.tar.gz - docker_arch: linux/arm64/v8 - profile: release - cross: true - - toolchain: - - stable - steps: - - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "test-daily-${{ matrix.platform.target }}" - - - name: Configure Git - run: | - git config --global user.email "jdoe@example.com" - git config --global user.name "J. Doe" - - - name: Install musl-tools, gnome-keyring and keyutils on Linux - run: | - sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils - rm -f $HOME/.local/share/keyrings/* - echo -n "test" | gnome-keyring-daemon --unlock - if: contains(matrix.platform.name, 'musl') - - - name: Prepare Cross.toml - run: | - scripts/prepare-cross-toml.sh - cat Cross.toml - if: ${{ matrix.platform.cross }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build binary ${{ matrix.platform.target }} - run: cross +stable build --features ci-qemu --verbose --target ${{ matrix.platform.target }} ${{ matrix.platform.profile == 'release' && '--release' || '' }} - - - name: Run tests ${{ matrix.platform.target }} - run: cross +stable test --features ci-qemu --verbose --target ${{ matrix.platform.target }} ${{ matrix.platform.profile == 'release' && '--release' || '' }} - - finalize_nightly: - runs-on: ubuntu-latest - needs: build_and_test - if: always() - steps: - - uses: actions/checkout@v4 - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name != 'workflow_dispatch' }} - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_BOT_CONTEXT_STRING: "nightly test suite" - with: - filename: .github/BOT_ISSUE_TEMPLATE.md diff --git a/.github/workflows/test_nightly.yml b/.github/workflows/test_nightly.yml deleted file mode 100644 index 11bbf2ce25..0000000000 --- a/.github/workflows/test_nightly.yml +++ /dev/null @@ -1,121 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: test_nightly -on: - workflow_dispatch: - schedule: - - cron: '0 23 * * *' # At 11:00 PM UTC, which is 10:00 PM CET - -env: - CRATE_NAME: iggy - GITHUB_TOKEN: ${{ github.token }} - RUST_BACKTRACE: 1 - CARGO_TERM_COLOR: always - IGGY_CI_BUILD: true - # Option needed for starting docker under cross to be able to lock memory - # in order to be able to use keyring inside Docker container - CROSS_CONTAINER_OPTS: "--cap-add ipc_lock" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build_and_test: - name: 'build and test ${{ matrix.toolchain }} ${{ matrix.platform.os_name }}' - runs-on: ${{ matrix.platform.os }} - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - platform: - - - os_name: Linux-x86_64-musl - os: ubuntu-latest - target: x86_64-unknown-linux-musl - name: iggy-Linux-x86_64-musl.tar.gz - cargo_command: cargo - profile: release - docker_arch: linux/amd64 - cross: false - - - os_name: Linux-aarch64-musl - os: ubuntu-latest - target: aarch64-unknown-linux-musl - name: iggy-Linux-aarch64-musl.tar.gz - docker_arch: linux/arm64/v8 - profile: release - cross: true - - toolchain: - - nightly - - beta - steps: - - uses: actions/checkout@v4 - - - name: Cache cargo & target directories - uses: Swatinem/rust-cache@v2 - with: - key: "test-nightly-${{ matrix.platform.target }}" - - - name: Configure Git - run: | - git config --global user.email "jdoe@example.com" - git config --global user.name "J. Doe" - - - name: Install musl-tools, gnome-keyring and keyutils on Linux - run: | - sudo apt-get update --yes && sudo apt-get install --yes musl-tools gnome-keyring keyutils - rm -f $HOME/.local/share/keyrings/* - echo -n "test" | gnome-keyring-daemon --unlock - if: contains(matrix.platform.name, 'musl') - - - name: Prepare Cross.toml - run: | - scripts/prepare-cross-toml.sh - cat Cross.toml - if: ${{ matrix.platform.cross }} - - - name: Prepare ${{ matrix.platform.target }} toolchain - run: | - rustup target add ${{ matrix.platform.target }} - echo "Using Rust toolchain from rust-toolchain.toml: $(rustup show)" - - - name: Install cross - uses: taiki-e/install-action@v2 - with: - tool: cross - - - name: Build binary ${{ matrix.platform.target }} - run: cross +stable build --features ci-qemu --verbose --target ${{ matrix.platform.target }} ${{ matrix.platform.profile == 'release' && '--release' || '' }} - - - name: Run tests ${{ matrix.platform.target }} - run: cross +stable test --features ci-qemu --verbose --target ${{ matrix.platform.target }} ${{ matrix.platform.profile == 'release' && '--release' || '' }} - - finalize_nightly: - runs-on: ubuntu-latest - needs: build_and_test - if: always() - steps: - - uses: actions/checkout@v4 - - name: Everything is fine - if: ${{ !(contains(needs.*.result, 'failure')) }} - run: exit 0 - - name: Something went wrong - if: ${{ contains(needs.*.result, 'failure') && github.event_name != 'workflow_dispatch' }} - run: exit 1 diff --git a/Cargo.lock b/Cargo.lock index dc1ec2656e..c26b8aa033 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -902,33 +902,6 @@ dependencies = [ "yew-router", ] -[[package]] -name = "bench-dashboard-server" -version = "0.4.0" -dependencies = [ - "actix-cors", - "actix-files", - "actix-web", - "bench-dashboard-shared", - "bench-report", - "chrono", - "clap", - "dashmap", - "file-operation", - "notify", - "octocrab", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.14", - "tokio", - "tracing", - "tracing-subscriber", - "uuid", - "walkdir", - "zip", -] - [[package]] name = "bench-dashboard-shared" version = "0.1.0" @@ -3847,6 +3820,33 @@ dependencies = [ "uuid", ] +[[package]] +name = "iggy-bench-dashboard-server" +version = "0.4.0" +dependencies = [ + "actix-cors", + "actix-files", + "actix-web", + "bench-dashboard-shared", + "bench-report", + "chrono", + "clap", + "dashmap", + "file-operation", + "notify", + "octocrab", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.14", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", + "walkdir", + "zip", +] + [[package]] name = "iggy-cli" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index b379d95d3b..8ecf29b5da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,10 +15,6 @@ # specific language governing permissions and limitations # under the License. -[profile.release] -lto = true -codegen-units = 1 - [workspace] members = [ "bdd/rust", @@ -49,6 +45,7 @@ exclude = ["foreign/python"] resolver = "2" [workspace.dependencies] +aes-gcm = "0.10.3" ahash = { version = "0.8.12", features = ["serde"] } anyhow = "1.0.99" async-broadcast = "0.7.2" @@ -64,7 +61,13 @@ async_zip = { version = "0.0.18", features = [ ] } axum = "0.8.4" axum-server = { version = "0.7.2", features = ["tls-rustls"] } +base64 = "0.22.1" bcrypt = "0.17.0" +bench-dashboard-frontend = { path = "core/bench/dashboard/frontend" } +bench-dashboard-server = { path = "core/bench/dashboard/server" } +bench-dashboard-shared = { path = "core/bench/dashboard/shared" } +bench-report = { path = "core/bench/report" } +bench-runner = { path = "core/bench/runner" } bincode = { version = "2.0.1", features = ["serde"] } blake3 = "1.8.2" bon = "3.7.0" @@ -77,50 +80,78 @@ bytes = "1.10.1" charming = "0.6.0" chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.44", features = ["derive"] } -config = { version = "0.15.13" } +colored = "3.0.0" comfy-table = "7.1.4" +config = { version = "0.15.13" } +console-subscriber = "0.4.1" crc32fast = "1.5.0" crossbeam = "0.8.4" dashmap = "6.1.0" +derive-new = "0.7.0" derive_builder = "0.20.2" derive_more = { version = "2.0.1", features = ["full"] } -derive-new = "0.7.0" dirs = "6.0.0" dlopen2 = "0.8.0" dotenvy = "0.15.7" enum_dispatch = "0.3.13" +env_logger = "0.11.8" figlet-rs = "0.1.5" flume = "0.11.1" futures = "0.3.31" futures-util = "0.3.31" human-repr = "1.1.0" humantime = "2.2.0" +iggy = { path = "core/sdk", version = "0.7.0" } +iggy_binary_protocol = { path = "core/binary_protocol", version = "0.7.0" } +iggy_common = { path = "core/common", version = "0.7.0" } +iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.1.0" } +integration = { path = "core/integration" } keyring = { version = "3.6.3", features = ["sync-secret-service", "vendored"] } +lazy_static = "1.5.0" +log = "0.4.27" +mimalloc = "0.1" +mockall = "0.13.1" nonzero_lit = "0.1.2" once_cell = "1.21.3" passterm = "=2.0.1" -quinn = "0.11.8" postcard = { version = "1.1.3", features = ["alloc"] } +predicates = "3.1.3" +quinn = "0.11.8" rand = "0.9.2" +regex = "1.11.1" reqwest = { version = "0.12.22", default-features = false, features = [ "json", "rustls-tls", ] } reqwest-middleware = { version = "0.4.2", features = ["json"] } reqwest-retry = "0.7.0" +rust-s3 = { version = "0.36.0-beta.2", default-features = false, features = [ + "tokio-rustls-tls", + "tags", +] } rustls = { version = "0.23.31", features = ["ring"] } serde = { version = "1.0.219", features = ["derive", "rc"] } serde_json = "1.0.142" serde_with = { version = "3.14.0", features = ["base64", "macros"] } serde_yml = "0.0.12" serial_test = "3.2.0" +server = { path = "core/server" } simd-json = { version = "0.15.1", features = ["serde_impl"] } +strum = { version = "0.27.2", features = ["derive"] } +strum_macros = "0.27.2" sysinfo = "0.37.0" tempfile = "3.20.0" +test-case = "3.3.1" thiserror = "2.0.14" tokio = { version = "1.47.1", features = ["full"] } tokio-rustls = "0.26.2" +tokio-util = { version = "0.7.16", features = ["compat"] } toml = "0.9.5" +tower-http = { version = "0.6.6", features = [ + "add-extension", + "cors", + "trace", +] } tracing = "0.1.41" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.19", default-features = false, features = [ @@ -128,6 +159,8 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features = "env-filter", "ansi", ] } +trait-variant = "0.1.2" +twox-hash = { version = "2.1.1", features = ["xxhash32"] } uuid = { version = "1.18.0", features = [ "v4", "v7", @@ -135,48 +168,9 @@ uuid = { version = "1.18.0", features = [ "serde", "zerocopy", ] } -rust-s3 = { version = "0.36.0-beta.2", default-features = false, features = [ - "tokio-rustls-tls", - "tags", -] } -strum = { version = "0.27.2", features = ["derive"] } -strum_macros = "0.27.2" -aes-gcm = "0.10.3" -base64 = "0.22.1" -twox-hash = { version = "2.1.1", features = ["xxhash32"] } - -# Common dependencies across multiple packages -colored = "3.0.0" -env_logger = "0.11.8" -lazy_static = "1.5.0" -log = "0.4.27" -mockall = "0.13.1" -predicates = "3.1.3" -regex = "1.11.1" -test-case = "3.3.1" -tokio-util = { version = "0.7.16", features = ["compat"] } -tower-http = { version = "0.6.6", features = [ - "add-extension", - "cors", - "trace", -] } -trait-variant = "0.1.2" webpki-roots = "1.0.2" zip = "4.3.0" -# Optional dependencies -mimalloc = "0.1" -console-subscriber = "0.4.1" - -# Path dependencies -iggy_binary_protocol = { path = "core/binary_protocol", version = "0.7.0" } -iggy_common = { path = "core/common", version = "0.7.0" } -iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.1.0" } -iggy = { path = "core/sdk", version = "0.7.0" } -server = { path = "core/server" } -integration = { path = "core/integration" } -bench-report = { path = "core/bench/report" } -bench-runner = { path = "core/bench/runner" } -bench-dashboard-frontend = { path = "core/bench/dashboard/frontend" } -bench-dashboard-server = { path = "core/bench/dashboard/server" } -bench-dashboard-shared = { path = "core/bench/dashboard/shared" } +[profile.release] +lto = true +codegen-units = 1 diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index ac93743e24..0000000000 --- a/Cross.toml +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This script is used to generate Cross.toml file for user which executes -# this script. This is needed since Cross.toml build.dockerfile.build-args -# section requires statically defined Docker build arguments and parameters -# like current UID or GID must be entered (cannot be generated or fetched -# during cross execution time). - -[build.env] -passthrough = ["IGGY_SYSTEM_PATH", "IGGY_CI_BUILD", "RUST_BACKTRACE=1"] diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 2e0dbc7477..8b37a37f1f 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -72,7 +72,6 @@ bcrypt: 0.17.0, "MIT", bdd: 0.0.1, "Apache-2.0", beef: 0.5.2, "Apache-2.0 OR MIT", bench-dashboard-frontend: 0.4.1, "Apache-2.0", -bench-dashboard-server: 0.4.0, "Apache-2.0", bench-dashboard-shared: 0.1.0, "Apache-2.0", bench-report: 0.2.2, "Apache-2.0", bench-runner: 0.1.0, "Apache-2.0", @@ -342,6 +341,7 @@ idna: 1.0.3, "Apache-2.0 OR MIT", idna_adapter: 1.2.1, "Apache-2.0 OR MIT", iggy: 0.7.0, "Apache-2.0", iggy-bench: 0.2.4, "Apache-2.0", +iggy-bench-dashboard-server: 0.4.0, "Apache-2.0", iggy-cli: 0.9.0, "Apache-2.0", iggy-connectors: 0.1.0, "Apache-2.0", iggy-mcp: 0.1.0, "Apache-2.0", diff --git a/Dockerfile.ci b/Dockerfile.ci deleted file mode 100644 index 1d5c6ed8f2..0000000000 --- a/Dockerfile.ci +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM alpine:latest - -ARG IGGY_CMD_PATH -RUN test -n "$IGGY_CMD_PATH" || (echo "IGGY_CMD_PATH not set" && false) - -ARG IGGY_SERVER_PATH -RUN test -n "$IGGY_SERVER_PATH" || (echo "IGGY_SERVER_PATH not set" && false) - -WORKDIR /iggy - -COPY ./core/configs ./configs -COPY ${IGGY_CMD_PATH} ./ -COPY ${IGGY_SERVER_PATH} ./ - -RUN chmod +x ./iggy -RUN chmod +x ./iggy-server - -RUN ln -sf /iggy/iggy /usr/bin/iggy - -CMD ["/iggy/iggy-server"] diff --git a/Dockerfile.connectors.ci b/Dockerfile.connectors.ci deleted file mode 100644 index b195c9141e..0000000000 --- a/Dockerfile.connectors.ci +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM alpine:latest - -ARG IGGY_CONNECTORS_PATH -RUN test -n "$IGGY_CONNECTORS_PATH" || (echo "IGGY_CONNECTORS_PATH not set" && false) - -WORKDIR /iggy - -COPY ./core/connectors/runtime/config.toml ./config.toml -COPY ${IGGY_CONNECTORS_PATH} ./ - -RUN chmod +x ./iggy-connectors - -RUN ln -sf /iggy/iggy-connectors /usr/bin/iggy-connectors - -CMD ["/iggy/iggy-connectors"] diff --git a/Dockerfile.cross b/Dockerfile.cross deleted file mode 100644 index 3c7e0ee08c..0000000000 --- a/Dockerfile.cross +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG CROSS_BASE_IMAGE -FROM $CROSS_BASE_IMAGE - -ARG USER -ARG CROSS_CONTAINER_UID -ARG CROSS_CONTAINER_GID - -USER root -RUN apt-get update \ - && apt-get install gnome-keyring \ - --yes --no-install-recommends \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Copy the entrypoint script into the container -COPY scripts/cross-docker-entrypoint.sh /usr/local/bin/ - -# Make the entrypoint script executable -RUN chmod +x /usr/local/bin/cross-docker-entrypoint.sh - -# Add user with specified UID and GID -RUN groupadd -g $CROSS_CONTAINER_GID $USER -RUN useradd -r -u $CROSS_CONTAINER_UID -g $CROSS_CONTAINER_GID -m $USER - -# Set the entry point -ENTRYPOINT ["/usr/local/bin/cross-docker-entrypoint.sh"] diff --git a/Dockerfile.cross.dockerignore b/Dockerfile.cross.dockerignore deleted file mode 100644 index 251cad5059..0000000000 --- a/Dockerfile.cross.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -.config -.github -/assets -/local_data -/certs -Dockerfile -docker-compose.yml -.dockerignore -.git -.gitignore diff --git a/Dockerfile.debug b/Dockerfile.debug deleted file mode 100644 index 412dda4384..0000000000 --- a/Dockerfile.debug +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Debug/test build for faster compilation during development and testing -FROM rust:1.89.0-alpine3.22 AS builder -RUN apk add musl-dev -WORKDIR /build -COPY . /build -# Build in debug mode for faster compilation times during testing -RUN cargo build --bin iggy-server - -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y \ - ca-certificates \ - liblzma5 \ - && rm -rf /var/lib/apt/lists/* -COPY ./core/configs ./configs -COPY --from=builder /build/target/debug/iggy-server . - -CMD ["/iggy-server"] diff --git a/Dockerfile.mcp.ci b/Dockerfile.mcp.ci deleted file mode 100644 index bc73ebe02f..0000000000 --- a/Dockerfile.mcp.ci +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM alpine:latest - -ARG IGGY_MCP_PATH -RUN test -n "$IGGY_MCP_PATH" || (echo "IGGY_MCP_PATH not set" && false) - -WORKDIR /iggy - -COPY ./core/ai/mcp/config.toml ./config.toml -COPY ${IGGY_MCP_PATH} ./ - -RUN chmod +x ./iggy-mcp - -RUN ln -sf /iggy/iggy-mcp /usr/bin/iggy-mcp - -CMD ["/iggy/iggy-mcp"] diff --git a/README.md b/README.md index e0f3a4f143..83b39bf578 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ The [Model Context Protocol](https://modelcontextprotocol.io) (MCP) is an open p The official Apache Iggy images can be found in [Docker Hub](https://hub.docker.com/r/apache/iggy), simply type `docker pull apache/iggy` to pull the image. -You can also find the images for all the different tooling such as Connectors, MCP Server etc. [here](https://hub.docker.com/u/apache?page=1&search=iggy). +You can also find the images for all the different tooling such as Connectors, MCP Server etc. at [Docker Hub](https://hub.docker.com/u/apache?page=1&search=iggy). Please note that the images tagged as `latest` are based on the official, stable releases, while the `edge` ones are updated directly from latest version of the `master` branch. diff --git a/bdd/docker-compose.yml b/bdd/docker-compose.yml index fb2a6c1a56..08313b9287 100644 --- a/bdd/docker-compose.yml +++ b/bdd/docker-compose.yml @@ -17,83 +17,95 @@ services: iggy-server: - image: bdd-iggy-server:latest + platform: linux/amd64 build: context: .. - dockerfile: bdd/Dockerfile + dockerfile: core/server/Dockerfile + target: runtime-prebuilt + args: + PREBUILT_IGGY_SERVER: ${IGGY_SERVER_PATH:-target/debug/iggy-server} + PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy} + LIBC: glibc + PROFILE: debug + command: [ "--fresh" ] environment: - RUST_LOG=info - IGGY_SYSTEM_PATH=local_data - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/stats"] - interval: 5s - timeout: 10s - retries: 10 - start_period: 30s volumes: - iggy_data:/app/local_data + networks: + - iggy-bdd-network rust-bdd: build: context: .. dockerfile: bdd/rust/Dockerfile depends_on: - iggy-server: - condition: service_healthy + - iggy-server environment: - IGGY_TCP_ADDRESS=iggy-server:8090 volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature - command: ["cargo", "test", "-p", "bdd", "--features", "iggy-server-in-docker", "--features", "bdd"] + command: [ "cargo", "test", "-p", "bdd", "--features", "iggy-server-in-docker", "--features", "bdd" ] + networks: + - iggy-bdd-network python-bdd: build: context: .. dockerfile: bdd/python/Dockerfile depends_on: - iggy-server: - condition: service_healthy + - iggy-server environment: - IGGY_TCP_ADDRESS=iggy-server:8090 volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature working_dir: /app - command: ["pytest", "tests/", "-v"] + command: [ "pytest", "tests/", "-v" ] + networks: + - iggy-bdd-network go-bdd: build: context: .. dockerfile: bdd/go/Dockerfile depends_on: - iggy-server: - condition: service_healthy + - iggy-server environment: - IGGY_TCP_ADDRESS=iggy-server:8090 volumes: - ./scenarios/basic_messaging.feature:/app/features/basic_messaging.feature command: [ "go", "test", "-v", "./..." ] + networks: + - iggy-bdd-network node-bdd: build: context: .. dockerfile: bdd/node/Dockerfile depends_on: - iggy-server: - condition: service_healthy + - iggy-server environment: - IGGY_TCP_ADDRESS=iggy-server:8090 - command: ["npm", "run", "test:bdd"] + command: [ "npm", "run", "test:bdd" ] + networks: + - iggy-bdd-network csharp-bdd: build: context: .. dockerfile: bdd/csharp/Dockerfile depends_on: - iggy-server: - condition: service_healthy + - iggy-server environment: - IGGY_TCP_ADDRESS=iggy-server:8090 - command: ["dotnet", "test"] - + command: [ "dotnet", "test" ] + networks: + - iggy-bdd-network + +networks: + iggy-bdd-network: + driver: bridge + volumes: iggy_data: diff --git a/bdd/python/Dockerfile b/bdd/python/Dockerfile index 389f5205d2..0241afdf65 100644 --- a/bdd/python/Dockerfile +++ b/bdd/python/Dockerfile @@ -15,12 +15,14 @@ # specific language governing permissions and limitations # under the License. -FROM rust:1.89-slim-bookworm +# syntax=docker/dockerfile:1 +ARG RUST_VERSION=1.89 +FROM rust:${RUST_VERSION}-slim-trixie RUN apt-get update && \ apt-get install -y \ - python3.11 \ - python3.11-dev \ + python3 \ + python3-dev \ python3-pip \ build-essential \ && rm -rf /var/lib/apt/lists/* @@ -39,7 +41,7 @@ WORKDIR /workspace/foreign/python RUN pip3 install --no-cache-dir -r /workspace/bdd/python/requirements.txt # Build and install the Iggy Python SDK -RUN maturin build --release --out dist && \ +RUN maturin build --out dist && \ pip3 install dist/*.whl # Set up BDD test directory diff --git a/bdd/python/tests/conftest.py b/bdd/python/tests/conftest.py index af3f2310a5..8cf4a8f555 100644 --- a/bdd/python/tests/conftest.py +++ b/bdd/python/tests/conftest.py @@ -28,6 +28,7 @@ @dataclass class GlobalContext: """Global test context similar to Rust implementation.""" + client: Optional[object] = None # Will be IggyClient server_addr: Optional[str] = None last_stream_id: Optional[int] = None diff --git a/bdd/rust/Dockerfile b/bdd/rust/Dockerfile index dc4df80742..63119a37b4 100644 --- a/bdd/rust/Dockerfile +++ b/bdd/rust/Dockerfile @@ -15,17 +15,19 @@ # specific language governing permissions and limitations # under the License. -FROM rust:1.89 +ARG RUST_VERSION=1.89 +FROM rust:${RUST_VERSION} WORKDIR /app -# Copy entire repository (needed for workspace) +# Copy everything COPY . . # Create features directory for BDD feature files RUN mkdir -p /app/features -RUN cargo fetch +# Build tests +RUN cargo test --no-run -p bdd --features "iggy-server-in-docker bdd" # Default command will be overridden by docker-compose CMD ["cargo", "test", "-p", "bdd", "--features", "iggy-server-in-docker", "--features", "bdd"] diff --git a/Dockerfile.ci.dockerignore b/core/ai/mcp/.dockerignore similarity index 100% rename from Dockerfile.ci.dockerignore rename to core/ai/mcp/.dockerignore diff --git a/core/ai/mcp/Dockerfile b/core/ai/mcp/Dockerfile index 7adbeaed16..6f42a6106a 100644 --- a/core/ai/mcp/Dockerfile +++ b/core/ai/mcp/Dockerfile @@ -1,31 +1,93 @@ # Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file +# or more contributor license agreements. See the NOTICE file # distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file +# regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the +# KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -FROM rust:1.89 AS builder -WORKDIR /build -COPY . /build -RUN cargo build --bin iggy-mcp --release +ARG RUST_VERSION=1.89 +ARG ALPINE_VERSION=3.22 -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y \ - ca-certificates \ - liblzma5 \ - && rm -rf /var/lib/apt/lists/* -COPY ./core/ai/mcp/config.toml ./config.toml -COPY --from=builder /build/target/release/iggy-mcp . +FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine AS chef +WORKDIR /app +RUN apk add --no-cache musl-dev pkgconfig openssl-dev openssl-libs-static -CMD ["/iggy-mcp"] +FROM --platform=$BUILDPLATFORM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM --platform=$BUILDPLATFORM chef AS builder +ARG PROFILE=release +ARG TARGETPLATFORM +ARG LIBC=musl + +RUN apk add --no-cache zig && \ + cargo install cargo-zigbuild --locked && \ + rustup target add \ + x86_64-unknown-linux-musl \ + aarch64-unknown-linux-musl \ + x86_64-unknown-linux-gnu \ + aarch64-unknown-linux-gnu + +COPY --from=planner /app/recipe.json recipe.json + +# +# Cook dependencies +# +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild; \ + else \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release; \ + fi + +COPY . . + +# +# Build +# +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-mcp && \ + cp /app/target/${RUST_TARGET}/debug/iggy-mcp /app/iggy-mcp; \ + else \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-mcp --release && \ + cp /app/target/${RUST_TARGET}/release/iggy-mcp /app/iggy-mcp; \ + fi + +# +# Final runtime - Debian trixie Slim +# +FROM debian:trixie-slim AS runtime +WORKDIR /app + +COPY --from=builder /app/iggy-mcp /usr/local/bin/iggy-mcp + +ENTRYPOINT ["iggy-mcp"] diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index db9a9bc8b6..d1a342be6d 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml @@ -25,6 +25,10 @@ homepage = "https://iggy.apache.org" description = "Benchmarking CLI for Iggy message streaming platform" keywords = ["iggy", "cli", "messaging", "streaming"] readme = "../../README.md" + +[[bin]] +name = "iggy-bench" +path = "src/main.rs" # Due to dependency to integration, which has a dependency to server, setting # mimalloc on server is also setting it on bench. @@ -54,10 +58,6 @@ tracing-appender = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true } -[[bin]] -name = "iggy-bench" -path = "src/main.rs" - [lints.clippy] enum_glob_use = "deny" pedantic = "deny" diff --git a/core/bench/dashboard/server/Cargo.toml b/core/bench/dashboard/server/Cargo.toml index 964f92a6c8..2743de7111 100644 --- a/core/bench/dashboard/server/Cargo.toml +++ b/core/bench/dashboard/server/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [package] -name = "bench-dashboard-server" +name = "iggy-bench-dashboard-server" license = "Apache-2.0" version = "0.4.0" edition = "2024" @@ -43,7 +43,3 @@ tracing-subscriber = { workspace = true } uuid = { workspace = true } walkdir = "2.5.0" zip = { workspace = true, features = ["deflate"] } - -[[bin]] -name = "iggy-bench-dashboard-server" -path = "src/main.rs" diff --git a/core/bench/dashboard/server/Dockerfile b/core/bench/dashboard/server/Dockerfile index 93306597ee..6673f19b4f 100644 --- a/core/bench/dashboard/server/Dockerfile +++ b/core/bench/dashboard/server/Dockerfile @@ -15,8 +15,12 @@ # specific language governing permissions and limitations # under the License. +ARG RUST_VERSION=1.89 + # Build stage -FROM rust:1.89-slim-bookworm AS builder +FROM rust:${RUST_VERSION}-slim-trixie AS builder +ARG IGGY_CI_BUILD +ENV IGGY_CI_BUILD=${IGGY_CI_BUILD} WORKDIR /usr/src/iggy-bench-dashboard @@ -47,10 +51,10 @@ COPY . . RUN cd core/bench/dashboard/frontend && trunk build --release # Build the server with release profile -RUN cargo build --release --package bench-dashboard-server +RUN cargo build --release --bin iggy-bench-dashboard-server # Runtime stage -FROM debian:bookworm-slim +FROM debian:trixie-slim WORKDIR /app diff --git a/core/bench/runner/Cargo.toml b/core/bench/runner/Cargo.toml index d078c64db4..3895197377 100644 --- a/core/bench/runner/Cargo.toml +++ b/core/bench/runner/Cargo.toml @@ -21,6 +21,10 @@ license = "Apache-2.0" version = "0.1.0" edition = "2024" +[[bin]] +name = "iggy-bench-runner" +path = "src/main.rs" + [dependencies] anyhow = { workspace = true } clap = { workspace = true } @@ -32,7 +36,3 @@ tempfile = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } - -[[bin]] -name = "iggy-bench-runner" -path = "src/main.rs" diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 78d1e59c31..90f1c3f7bc 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml @@ -31,6 +31,10 @@ readme = "../../README.md" pkg-url = "{ repo }/releases/download/{ name }-{ version }/{ name }-{ target }{ archive-suffix }" bin-dir = "{ bin }{ binary-ext }" +[[bin]] +name = "iggy" +path = "src/main.rs" + [features] default = ["login-session"] login-session = ["dep:keyring"] @@ -50,7 +54,3 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-appender = { workspace = true } tracing-subscriber = { workspace = true, default-features = false } - -[[bin]] -name = "iggy" -path = "src/main.rs" diff --git a/Dockerfile.connectors.ci.dockerignore b/core/connectors/runtime/.dockerignore similarity index 100% rename from Dockerfile.connectors.ci.dockerignore rename to core/connectors/runtime/.dockerignore diff --git a/core/connectors/runtime/Dockerfile b/core/connectors/runtime/Dockerfile index bf4ef2c302..67311ce205 100644 --- a/core/connectors/runtime/Dockerfile +++ b/core/connectors/runtime/Dockerfile @@ -1,31 +1,93 @@ # Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file +# or more contributor license agreements. See the NOTICE file # distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file +# regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the +# KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -FROM rust:1.89 AS builder -WORKDIR /build -COPY . /build -RUN cargo build --bin iggy-connectors --release +ARG RUST_VERSION=1.89 +ARG ALPINE_VERSION=3.22 -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y \ - ca-certificates \ - liblzma5 \ - && rm -rf /var/lib/apt/lists/* -COPY ./core/connectors/runtime/config.toml ./config.toml -COPY --from=builder /build/target/release/iggy-connectors . +FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine AS chef +WORKDIR /app +RUN apk add --no-cache musl-dev pkgconfig openssl-dev openssl-libs-static -CMD ["/iggy-connectors"] +FROM --platform=$BUILDPLATFORM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM --platform=$BUILDPLATFORM chef AS builder +ARG PROFILE=release +ARG TARGETPLATFORM +ARG LIBC=musl + +RUN apk add --no-cache zig && \ + cargo install cargo-zigbuild --locked && \ + rustup target add \ + x86_64-unknown-linux-musl \ + aarch64-unknown-linux-musl \ + x86_64-unknown-linux-gnu \ + aarch64-unknown-linux-gnu + +COPY --from=planner /app/recipe.json recipe.json + +# +# Cook dependencies +# +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild; \ + else \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release; \ + fi + +COPY . . + +# +# Build +# +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-connectors && \ + cp /app/target/${RUST_TARGET}/debug/iggy-connectors /app/iggy-connectors; \ + else \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-connectors --release && \ + cp /app/target/${RUST_TARGET}/release/iggy-connectors /app/iggy-connectors; \ + fi + +# +# Final runtime - Debian trixie Slim +# +FROM debian:trixie-slim AS runtime +WORKDIR /app + +COPY --from=builder /app/iggy-connectors /usr/local/bin/iggy-connectors + +ENTRYPOINT ["iggy-connectors"] diff --git a/core/connectors/sinks/quickwit_sink/Cargo.toml b/core/connectors/sinks/quickwit_sink/Cargo.toml index 3537cbbd68..cdf402a8d0 100644 --- a/core/connectors/sinks/quickwit_sink/Cargo.toml +++ b/core/connectors/sinks/quickwit_sink/Cargo.toml @@ -31,6 +31,9 @@ readme = "../../README.md" [package.metadata.cargo-machete] ignored = ["dashmap", "once_cell"] +[lib] +crate-type = ["cdylib", "lib"] + [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } @@ -41,6 +44,3 @@ serde = { workspace = true } serde_yml = { workspace = true } simd-json = { workspace = true } tracing = { workspace = true } - -[lib] -crate-type = ["cdylib", "lib"] diff --git a/core/connectors/sinks/stdout_sink/Cargo.toml b/core/connectors/sinks/stdout_sink/Cargo.toml index 5ad4fa402e..2797e04ba0 100644 --- a/core/connectors/sinks/stdout_sink/Cargo.toml +++ b/core/connectors/sinks/stdout_sink/Cargo.toml @@ -31,6 +31,9 @@ readme = "../../README.md" [package.metadata.cargo-machete] ignored = ["dashmap", "once_cell"] +[lib] +crate-type = ["cdylib", "lib"] + [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } @@ -39,6 +42,3 @@ once_cell = { workspace = true } serde = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } - -[lib] -crate-type = ["cdylib", "lib"] diff --git a/core/connectors/sources/random_source/Cargo.toml b/core/connectors/sources/random_source/Cargo.toml index 53d17933b5..ad645601dd 100644 --- a/core/connectors/sources/random_source/Cargo.toml +++ b/core/connectors/sources/random_source/Cargo.toml @@ -31,6 +31,9 @@ readme = "../../README.md" [package.metadata.cargo-machete] ignored = ["dashmap", "once_cell"] +[lib] +crate-type = ["cdylib", "lib"] + [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } @@ -43,6 +46,3 @@ simd-json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } - -[lib] -crate-type = ["cdylib", "lib"] diff --git a/core/integration/tests/cli/common/mod.rs b/core/integration/tests/cli/common/mod.rs index 077c8aa470..3bb3743a05 100644 --- a/core/integration/tests/cli/common/mod.rs +++ b/core/integration/tests/cli/common/mod.rs @@ -145,6 +145,8 @@ impl IggyCmdTest { let command_args = test_case.get_command(); // Set environment variables for the command command.envs(command_args.get_env()); + // Set a fixed terminal width for consistent help output formatting + command.env("COLUMNS", "500"); // Set server address for the command - it's randomized for each test command.args(test_case.protocol(&self.server)); @@ -205,6 +207,8 @@ impl IggyCmdTest { let command_args = test_case.get_command(); // Set environment variables for the command command.envs(command_args.get_env()); + // Set a fixed terminal width for consistent help output formatting + command.env("COLUMNS", "500"); // Print used environment variables and command with all arguments. // By default, it will not be visible but once test is executed with diff --git a/core/integration/tests/cli/topic/test_topic_create_command.rs b/core/integration/tests/cli/topic/test_topic_create_command.rs index 0227d79cc4..f4dc9ad565 100644 --- a/core/integration/tests/cli/topic/test_topic_create_command.rs +++ b/core/integration/tests/cli/topic/test_topic_create_command.rs @@ -350,14 +350,10 @@ Arguments: [MESSAGE_EXPIRY]... Message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default] Options: - -t, --topic-id - Topic ID to create - -m, --max-topic-size - Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] - -r, --replication-factor - Replication factor for the topic [default: 1] - -h, --help - Print help (see more with '--help') + -t, --topic-id Topic ID to create + -m, --max-topic-size Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] + -r, --replication-factor Replication factor for the topic [default: 1] + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/core/integration/tests/cli/topic/test_topic_update_command.rs b/core/integration/tests/cli/topic/test_topic_update_command.rs index d250f4045e..ac402345d6 100644 --- a/core/integration/tests/cli/topic/test_topic_update_command.rs +++ b/core/integration/tests/cli/topic/test_topic_update_command.rs @@ -457,12 +457,9 @@ Arguments: [MESSAGE_EXPIRY]... New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default] Options: - -m, --max-topic-size - New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] - -r, --replication-factor - New replication factor for the topic [default: 1] - -h, --help - Print help (see more with '--help') + -m, --max-topic-size New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] + -r, --replication-factor New replication factor for the topic [default: 1] + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/core/integration/tests/cli/user/test_user_create_command.rs b/core/integration/tests/cli/user/test_user_create_command.rs index ecae4c4744..3678185bd9 100644 --- a/core/integration/tests/cli/user/test_user_create_command.rs +++ b/core/integration/tests/cli/user/test_user_create_command.rs @@ -381,14 +381,10 @@ Arguments: Password Options: - -u, --user-status - User status [default: active] [possible values: active, inactive] - -g, --global-permissions - Set global permissions for created user - -s, --stream-permissions - Set stream permissions for created user - -h, --help - Print help (see more with '--help') + -u, --user-status User status [default: active] [possible values: active, inactive] + -g, --global-permissions Set global permissions for created user + -s, --stream-permissions Set stream permissions for created user + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/core/integration/tests/cli/user/test_user_permissions_command.rs b/core/integration/tests/cli/user/test_user_permissions_command.rs index c4cf30cb7d..247641ef34 100644 --- a/core/integration/tests/cli/user/test_user_permissions_command.rs +++ b/core/integration/tests/cli/user/test_user_permissions_command.rs @@ -346,14 +346,10 @@ Arguments: Password Options: - -u, --user-status - User status [default: active] [possible values: active, inactive] - -g, --global-permissions - Set global permissions for created user - -s, --stream-permissions - Set stream permissions for created user - -h, --help - Print help (see more with '--help') + -u, --user-status User status [default: active] [possible values: active, inactive] + -g, --global-permissions Set global permissions for created user + -s, --stream-permissions Set stream permissions for created user + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/Dockerfile.mcp.ci.dockerignore b/core/server/.dockerignore similarity index 100% rename from Dockerfile.mcp.ci.dockerignore rename to core/server/.dockerignore diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index 3a0cf94e9c..24d2b31767 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml @@ -28,6 +28,10 @@ normal = ["tracing-appender"] [package.metadata.cargo-machete] ignored = ["rust-s3"] +[[bin]] +name = "iggy-server" +path = "src/main.rs" + [features] default = ["mimalloc"] tokio-console = ["dep:console-subscriber", "tokio/tracing"] @@ -125,7 +129,3 @@ vergen-git2 = { version = "1.0.7", features = [ [dev-dependencies] mockall = { workspace = true } serial_test = { workspace = true } - -[[bin]] -name = "iggy-server" -path = "src/main.rs" diff --git a/core/server/Dockerfile b/core/server/Dockerfile new file mode 100644 index 0000000000..f43b82dee2 --- /dev/null +++ b/core/server/Dockerfile @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG RUST_VERSION=1.89 +ARG ALPINE_VERSION=3.22 + +# ── from-source path (unchanged) ────────────────────────────────────────────── +FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine AS chef +WORKDIR /app +RUN apk add --no-cache musl-dev pkgconfig openssl-dev openssl-libs-static + +FROM --platform=$BUILDPLATFORM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM --platform=$BUILDPLATFORM chef AS builder +ARG PROFILE=release +ARG TARGETPLATFORM +ARG LIBC=musl + +RUN apk add --no-cache zig && \ + cargo install cargo-zigbuild --locked && \ + rustup target add \ + x86_64-unknown-linux-musl \ + aarch64-unknown-linux-musl \ + x86_64-unknown-linux-gnu \ + aarch64-unknown-linux-gnu + +COPY --from=planner /app/recipe.json recipe.json + +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild; \ + else \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release; \ + fi + +COPY . . + +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-server --bin iggy && \ + cp /app/target/${RUST_TARGET}/debug/iggy-server /app/iggy-server && \ + cp /app/target/${RUST_TARGET}/debug/iggy /app/iggy; \ + else \ + cargo zigbuild --target ${RUST_TARGET} --bin iggy-server --bin iggy --release && \ + cp /app/target/${RUST_TARGET}/release/iggy-server /app/iggy-server && \ + cp /app/target/${RUST_TARGET}/release/iggy /app/iggy; \ + fi + +# ── prebuilt path (FAST) ────────────────────────────────────────────────────── +FROM debian:trixie-slim AS prebuilt +WORKDIR /out +ARG PREBUILT_IGGY_SERVER +ARG PREBUILT_IGGY_CLI +COPY ${PREBUILT_IGGY_SERVER} /out/iggy-server +COPY ${PREBUILT_IGGY_CLI} /out/iggy +RUN chmod +x /out/iggy-server /out/iggy + +# ── final images ────────────────────────────────────────────────────────────── +FROM debian:trixie-slim AS runtime-prebuilt +ARG TARGETPLATFORM +ARG PREBUILT_IGGY_SERVER +ARG PREBUILT_IGGY_CLI +WORKDIR /app +COPY --from=prebuilt /out/iggy-server /usr/local/bin/iggy-server +COPY --from=prebuilt /out/iggy /usr/local/bin/iggy +RUN echo "═══════════════════════════════════════════════════════════════" && \ + echo " IGGY SERVER BUILD SUMMARY " && \ + echo "═══════════════════════════════════════════════════════════════" && \ + echo "Build Type: PREBUILT BINARIES" && \ + echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ + echo "Source Path: ${PREBUILT_IGGY_SERVER:-not specified}" && \ + echo "Binary Info:" && \ + (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server | sed 's/^/ /' || \ + echo " $(ldd /usr/local/bin/iggy-server 2>&1 | head -1)") && \ + echo "Binary Size:" && \ + ls -lh /usr/local/bin/iggy-server /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ + echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ + echo "Container Base: debian:trixie-slim" && \ + echo "═══════════════════════════════════════════════════════════════" +ENTRYPOINT ["iggy-server"] + +FROM debian:trixie-slim AS runtime +ARG TARGETPLATFORM +ARG PROFILE=release +ARG LIBC=musl +WORKDIR /app +COPY --from=builder /app/iggy-server /usr/local/bin/iggy-server +COPY --from=builder /app/iggy /usr/local/bin/iggy +RUN echo "═══════════════════════════════════════════════════════════════" && \ + echo " IGGY SERVER BUILD SUMMARY " && \ + echo "═══════════════════════════════════════════════════════════════" && \ + echo "Build Type: FROM SOURCE" && \ + echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ + echo "Profile: ${PROFILE}" && \ + echo "Libc: ${LIBC}" && \ + case "${TARGETPLATFORM:-linux/amd64}:${LIBC}" in \ + "linux/amd64:musl") echo "Target: x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") echo "Target: aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") echo "Target: x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") echo "Target: aarch64-unknown-linux-gnu" ;; \ + *) echo "Target: unknown" ;; \ + esac && \ + echo "Binary Info:" && \ + (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server | sed 's/^/ /' || \ + echo " $(ldd /usr/local/bin/iggy-server 2>&1 | head -1)") && \ + echo "Binary Size:" && \ + ls -lh /usr/local/bin/iggy-server /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ + echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ + echo "═══════════════════════════════════════════════════════════════" +ENTRYPOINT ["iggy-server"] diff --git a/core/tools/Cargo.toml b/core/tools/Cargo.toml index be036b36a4..55cc857e24 100644 --- a/core/tools/Cargo.toml +++ b/core/tools/Cargo.toml @@ -21,6 +21,10 @@ version = "0.1.0" edition = "2024" license = "Apache-2.0" +[[bin]] +name = "data-seeder-tool" +path = "src/data-seeder/main.rs" + [dependencies] anyhow = { workspace = true } clap = { workspace = true } @@ -29,7 +33,3 @@ rand = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } - -[[bin]] -name = "data-seeder-tool" -path = "src/data-seeder/main.rs" diff --git a/docker-compose.yml b/docker-compose.yml index 48da4ee0e6..3e485c1064 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,9 @@ services: iggy-server: - build: . + build: + context: . + dockerfile: core/server/Dockerfile container_name: iggy-server restart: unless-stopped networks: diff --git a/examples/python/getting-started/consumer.py b/examples/python/getting-started/consumer.py index efd7bcbdd7..37cb51e690 100644 --- a/examples/python/getting-started/consumer.py +++ b/examples/python/getting-started/consumer.py @@ -15,15 +15,15 @@ # specific language governing permissions and limitations # under the License. -import asyncio -import urllib.parse -from loguru import logger import argparse -import urllib +import asyncio import typing +import urllib +import urllib.parse from collections import namedtuple -from apache_iggy import IggyClient, ReceiveMessage, PollingStrategy +from apache_iggy import IggyClient, PollingStrategy, ReceiveMessage +from loguru import logger STREAM_NAME = "sample-stream" TOPIC_NAME = "sample-topic" diff --git a/examples/python/getting-started/producer.py b/examples/python/getting-started/producer.py index be20db629d..71ccb22324 100644 --- a/examples/python/getting-started/producer.py +++ b/examples/python/getting-started/producer.py @@ -15,15 +15,16 @@ # specific language governing permissions and limitations # under the License. -import asyncio -import urllib.parse -from loguru import logger import argparse -import urllib +import asyncio import typing +import urllib +import urllib.parse from collections import namedtuple -from apache_iggy import IggyClient, SendMessage as Message, StreamDetails, TopicDetails +from apache_iggy import IggyClient, StreamDetails, TopicDetails +from apache_iggy import SendMessage as Message +from loguru import logger STREAM_NAME = "sample-stream" TOPIC_NAME = "sample-topic" diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index 2d523c4cb2..51c1a9d7c9 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs @@ -112,7 +112,7 @@ public void MapMessagesTMessage_NoHeaders_ReturnsValidMessageResponse() Text = text }; }); - //Assert + //Assert Assert.NotEmpty(response.Messages); Assert.Equal(2, response.Messages.Count); // Assert.Equal(response.Messages[0].Id, guid); @@ -397,4 +397,4 @@ public void MapStats_ReturnsValidStatsResponse() Assert.Equal(stats.OsVersion, stats.OsVersion); Assert.Equal(stats.KernelVersion, response.KernelVersion); } -} \ No newline at end of file +} diff --git a/foreign/csharp/scripts/pack.sh b/foreign/csharp/scripts/pack.sh index b2d87f80ce..6510e6ac28 100755 --- a/foreign/csharp/scripts/pack.sh +++ b/foreign/csharp/scripts/pack.sh @@ -39,14 +39,14 @@ echo "Extracted version: $version" echo "Executing after success scripts on branch $GITHUB_REF_NAME" echo "Triggering Nuget package build" -cd Iggy_SDK -dotnet pack -c release /p:PackageVersion=$version --no-restore -o . +cd Iggy_SDK || exit +dotnet pack -c release /p:PackageVersion="$version" --no-restore -o . echo "Uploading Iggy package to Nuget using branch $GITHUB_REF_NAME" case "$GITHUB_REF_NAME" in "master") - dotnet nuget push *.nupkg -k $NUGET_API_KEY -s https://api.nuget.org/v3/index.json + dotnet nuget push ./*.nupkg -k "$NUGET_API_KEY" -s https://api.nuget.org/v3/index.json echo "Published package succesfully!" ;; *) diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index c7300a5765..801f05960a 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -32,70 +32,86 @@ class AutoCommit: r""" The auto-commit is disabled and the offset must be stored manually by the consumer. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommit.Disabled: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class Interval(AutoCommit): r""" The auto-commit is enabled and the offset is stored on the server after a certain interval. """ + __match_args__ = ("_0",) @property def _0(self) -> datetime.timedelta: ... - def __new__(cls, _0:datetime.timedelta) -> AutoCommit.Interval: ... + def __new__(cls, _0: datetime.timedelta) -> AutoCommit.Interval: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class IntervalOrWhen(AutoCommit): r""" The auto-commit is enabled and the offset is stored on the server after a certain interval or depending on the mode when consuming the messages. """ - __match_args__ = ("_0", "_1",) + + __match_args__ = ( + "_0", + "_1", + ) @property def _0(self) -> datetime.timedelta: ... @property def _1(self) -> AutoCommitWhen: ... - def __new__(cls, _0:datetime.timedelta, _1:AutoCommitWhen) -> AutoCommit.IntervalOrWhen: ... + def __new__( + cls, _0: datetime.timedelta, _1: AutoCommitWhen + ) -> AutoCommit.IntervalOrWhen: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class IntervalOrAfter(AutoCommit): r""" The auto-commit is enabled and the offset is stored on the server after a certain interval or depending on the mode after consuming the messages. """ - __match_args__ = ("_0", "_1",) + + __match_args__ = ( + "_0", + "_1", + ) @property def _0(self) -> datetime.timedelta: ... @property def _1(self) -> AutoCommitAfter: ... - def __new__(cls, _0:datetime.timedelta, _1:AutoCommitAfter) -> AutoCommit.IntervalOrAfter: ... + def __new__( + cls, _0: datetime.timedelta, _1: AutoCommitAfter + ) -> AutoCommit.IntervalOrAfter: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class When(AutoCommit): r""" The auto-commit is enabled and the offset is stored on the server depending on the mode when consuming the messages. """ + __match_args__ = ("_0",) @property def _0(self) -> AutoCommitWhen: ... - def __new__(cls, _0:AutoCommitWhen) -> AutoCommit.When: ... + def __new__(cls, _0: AutoCommitWhen) -> AutoCommit.When: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class After(AutoCommit): r""" The auto-commit is enabled and the offset is stored on the server depending on the mode after consuming the messages. """ + __match_args__ = ("_0",) @property def _0(self) -> AutoCommitAfter: ... - def __new__(cls, _0:AutoCommitAfter) -> AutoCommit.After: ... + def __new__(cls, _0: AutoCommitAfter) -> AutoCommit.After: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + ... class AutoCommitAfter: @@ -106,31 +122,36 @@ class AutoCommitAfter: r""" The offset is stored on the server after all the messages are consumed. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommitAfter.ConsumingAllMessages: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class ConsumingEachMessage(AutoCommitAfter): r""" The offset is stored on the server after consuming each message. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommitAfter.ConsumingEachMessage: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class ConsumingEveryNthMessage(AutoCommitAfter): r""" The offset is stored on the server after consuming every Nth message. """ + __match_args__ = ("_0",) @property def _0(self) -> builtins.int: ... - def __new__(cls, _0:builtins.int) -> AutoCommitAfter.ConsumingEveryNthMessage: ... + def __new__( + cls, _0: builtins.int + ) -> AutoCommitAfter.ConsumingEveryNthMessage: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + ... class AutoCommitWhen: @@ -141,40 +162,46 @@ class AutoCommitWhen: r""" The offset is stored on the server when the messages are received. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommitWhen.PollingMessages: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class ConsumingAllMessages(AutoCommitWhen): r""" The offset is stored on the server when all the messages are consumed. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommitWhen.ConsumingAllMessages: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class ConsumingEachMessage(AutoCommitWhen): r""" The offset is stored on the server when consuming each message. """ + __match_args__ = ((),) def __new__(cls) -> AutoCommitWhen.ConsumingEachMessage: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + class ConsumingEveryNthMessage(AutoCommitWhen): r""" The offset is stored on the server when consuming every Nth message. """ + __match_args__ = ("_0",) @property def _0(self) -> builtins.int: ... - def __new__(cls, _0:builtins.int) -> AutoCommitWhen.ConsumingEveryNthMessage: ... + def __new__( + cls, _0: builtins.int + ) -> AutoCommitWhen.ConsumingEveryNthMessage: ... def __len__(self) -> builtins.int: ... - def __getitem__(self, key:builtins.int) -> typing.Any: ... - + def __getitem__(self, key: builtins.int) -> typing.Any: ... + ... class IggyClient: @@ -183,79 +210,127 @@ class IggyClient: It wraps the RustIggyClient and provides asynchronous functionality through the contained runtime. """ - def __new__(cls, conn:typing.Optional[builtins.str]=None) -> IggyClient: + def __new__(cls, conn: typing.Optional[builtins.str] = None) -> IggyClient: r""" Constructs a new IggyClient from a TCP server address. - + This initializes a new runtime for asynchronous operations. Future versions might utilize asyncio for more Pythonic async. """ @classmethod - def from_connection_string(cls, connection_string:builtins.str) -> IggyClient: + def from_connection_string(cls, connection_string: builtins.str) -> IggyClient: r""" Constructs a new IggyClient from a connection string. - + Returns an error if the connection string provided is invalid. """ def ping(self) -> collections.abc.Awaitable[None]: r""" Sends a ping request to the server to check connectivity. - + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` if the connection fails. """ - def login_user(self, username:builtins.str, password:builtins.str) -> collections.abc.Awaitable[None]: + def login_user( + self, username: builtins.str, password: builtins.str + ) -> collections.abc.Awaitable[None]: r""" Logs in the user with the given credentials. - + Returns `Ok(())` on success, or a PyRuntimeError on failure. """ def connect(self) -> collections.abc.Awaitable[None]: r""" Connects the IggyClient to its service. - + Returns Ok(()) on successful connection or a PyRuntimeError on failure. """ - def create_stream(self, name:builtins.str, stream_id:typing.Optional[builtins.int]=None) -> collections.abc.Awaitable[None]: + def create_stream( + self, name: builtins.str, stream_id: typing.Optional[builtins.int] = None + ) -> collections.abc.Awaitable[None]: r""" Creates a new stream with the provided ID and name. - + Returns Ok(()) on successful stream creation or a PyRuntimeError on failure. """ - def get_stream(self, stream_id:builtins.str | builtins.int) -> collections.abc.Awaitable[typing.Optional[StreamDetails]]: + def get_stream( + self, stream_id: builtins.str | builtins.int + ) -> collections.abc.Awaitable[typing.Optional[StreamDetails]]: r""" Gets stream by id. - + Returns Option of stream details or a PyRuntimeError on failure. """ - def create_topic(self, stream:builtins.str | builtins.int, name:builtins.str, partitions_count:builtins.int, compression_algorithm:typing.Optional[builtins.str]=None, topic_id:typing.Optional[builtins.int]=None, replication_factor:typing.Optional[builtins.int]=None) -> collections.abc.Awaitable[None]: + def create_topic( + self, + stream: builtins.str | builtins.int, + name: builtins.str, + partitions_count: builtins.int, + compression_algorithm: typing.Optional[builtins.str] = None, + topic_id: typing.Optional[builtins.int] = None, + replication_factor: typing.Optional[builtins.int] = None, + ) -> collections.abc.Awaitable[None]: r""" Creates a new topic with the given parameters. - + Returns Ok(()) on successful topic creation or a PyRuntimeError on failure. """ - def get_topic(self, stream_id:builtins.str | builtins.int, topic_id:builtins.str | builtins.int) -> collections.abc.Awaitable[typing.Optional[TopicDetails]]: + def get_topic( + self, + stream_id: builtins.str | builtins.int, + topic_id: builtins.str | builtins.int, + ) -> collections.abc.Awaitable[typing.Optional[TopicDetails]]: r""" Gets topic by stream and id. - + Returns Option of topic details or a PyRuntimeError on failure. """ - def send_messages(self, stream:builtins.str | builtins.int, topic:builtins.str | builtins.int, partitioning:builtins.int, messages:list[SendMessage]) -> collections.abc.Awaitable[None]: + def send_messages( + self, + stream: builtins.str | builtins.int, + topic: builtins.str | builtins.int, + partitioning: builtins.int, + messages: list[SendMessage], + ) -> collections.abc.Awaitable[None]: r""" Sends a list of messages to the specified topic. - + Returns Ok(()) on successful sending or a PyRuntimeError on failure. """ - def poll_messages(self, stream:builtins.str | builtins.int, topic:builtins.str | builtins.int, partition_id:builtins.int, polling_strategy:PollingStrategy, count:builtins.int, auto_commit:builtins.bool) -> collections.abc.Awaitable[list[ReceiveMessage]]: + def poll_messages( + self, + stream: builtins.str | builtins.int, + topic: builtins.str | builtins.int, + partition_id: builtins.int, + polling_strategy: PollingStrategy, + count: builtins.int, + auto_commit: builtins.bool, + ) -> collections.abc.Awaitable[list[ReceiveMessage]]: r""" Polls for messages from the specified topic and partition. - + Returns a list of received messages or a PyRuntimeError on failure. """ - def consumer_group(self, name:builtins.str, stream:builtins.str, topic:builtins.str, partition_id:typing.Optional[builtins.int]=None, polling_strategy:typing.Optional[PollingStrategy]=None, batch_length:typing.Optional[builtins.int]=None, auto_commit:typing.Optional[AutoCommit]=None, create_consumer_group_if_not_exists:builtins.bool=True, auto_join_consumer_group:builtins.bool=True, poll_interval:typing.Optional[datetime.timedelta]=None, polling_retry_interval:typing.Optional[datetime.timedelta]=None, init_retries:typing.Optional[builtins.int]=None, init_retry_interval:typing.Optional[datetime.timedelta]=None, allow_replay:builtins.bool=False) -> IggyConsumer: + def consumer_group( + self, + name: builtins.str, + stream: builtins.str, + topic: builtins.str, + partition_id: typing.Optional[builtins.int] = None, + polling_strategy: typing.Optional[PollingStrategy] = None, + batch_length: typing.Optional[builtins.int] = None, + auto_commit: typing.Optional[AutoCommit] = None, + create_consumer_group_if_not_exists: builtins.bool = True, + auto_join_consumer_group: builtins.bool = True, + poll_interval: typing.Optional[datetime.timedelta] = None, + polling_retry_interval: typing.Optional[datetime.timedelta] = None, + init_retries: typing.Optional[builtins.int] = None, + init_retry_interval: typing.Optional[datetime.timedelta] = None, + allow_replay: builtins.bool = False, + ) -> IggyConsumer: r""" Creates a new consumer group consumer. - + Returns the consumer or a PyRuntimeError on failure. """ @@ -265,11 +340,15 @@ class IggyConsumer: It wraps the RustIggyConsumer and provides asynchronous functionality through the contained runtime. """ - def get_last_consumed_offset(self, partition_id:builtins.int) -> typing.Optional[builtins.int]: + def get_last_consumed_offset( + self, partition_id: builtins.int + ) -> typing.Optional[builtins.int]: r""" Get the last consumed offset or `None` if no offset has been consumed yet. """ - def get_last_stored_offset(self, partition_id:builtins.int) -> typing.Optional[builtins.int]: + def get_last_stored_offset( + self, partition_id: builtins.int + ) -> typing.Optional[builtins.int]: r""" Get the last stored offset or `None` if no offset has been stored yet. """ @@ -289,26 +368,36 @@ class IggyConsumer: r""" Gets the name of the topic this consumer group is configured for. """ - def store_offset(self, offset:builtins.int, partition_id:typing.Optional[builtins.int]) -> collections.abc.Awaitable[None]: + def store_offset( + self, offset: builtins.int, partition_id: typing.Optional[builtins.int] + ) -> collections.abc.Awaitable[None]: r""" Stores the provided offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` if the operation fails. """ - def delete_offset(self, partition_id:typing.Optional[builtins.int]) -> collections.abc.Awaitable[None]: + def delete_offset( + self, partition_id: typing.Optional[builtins.int] + ) -> collections.abc.Awaitable[None]: r""" Deletes the offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` if the operation fails. """ - def consume_messages(self, callback:collections.abc.Callable[[ReceiveMessage], collections.abc.Awaitable[None]], shutdown_event:typing.Optional[asyncio.Event]) -> collections.abc.Awaitable[None]: + def consume_messages( + self, + callback: collections.abc.Callable[ + [ReceiveMessage], collections.abc.Awaitable[None] + ], + shutdown_event: typing.Optional[asyncio.Event], + ) -> collections.abc.Awaitable[None]: r""" Consumes messages continuously using a callback function and an optional `asyncio.Event` for signaling shutdown. - + Returns an awaitable that completes when shutdown is signaled or a PyRuntimeError on failure. """ @@ -317,82 +406,82 @@ class PollingStrategy: __match_args__ = ("value",) @property def value(self) -> builtins.int: ... - def __new__(cls, value:builtins.int) -> PollingStrategy.Offset: ... - + def __new__(cls, value: builtins.int) -> PollingStrategy.Offset: ... + class Timestamp(PollingStrategy): __match_args__ = ("value",) @property def value(self) -> builtins.int: ... - def __new__(cls, value:builtins.int) -> PollingStrategy.Timestamp: ... - + def __new__(cls, value: builtins.int) -> PollingStrategy.Timestamp: ... + class First(PollingStrategy): __match_args__ = ((),) def __new__(cls) -> PollingStrategy.First: ... - + class Last(PollingStrategy): __match_args__ = ((),) def __new__(cls) -> PollingStrategy.Last: ... - + class Next(PollingStrategy): __match_args__ = ((),) def __new__(cls) -> PollingStrategy.Next: ... - + ... class ReceiveMessage: r""" A Python class representing a received message. - + This class wraps a Rust message, allowing for access to its payload and offset from Python. """ def payload(self) -> bytes: r""" Retrieves the payload of the received message. - + The payload is returned as a Python bytes object. """ def offset(self) -> builtins.int: r""" Retrieves the offset of the received message. - + The offset represents the position of the message within its topic. """ def timestamp(self) -> builtins.int: r""" Retrieves the timestamp of the received message. - + The timestamp represents the time of the message within its topic. """ def id(self) -> builtins.int: r""" Retrieves the id of the received message. - + The id represents unique identifier of the message within its topic. """ def checksum(self) -> builtins.int: r""" Retrieves the checksum of the received message. - + The checksum represents the integrity of the message within its topic. """ def length(self) -> builtins.int: r""" Retrieves the length of the received message. - + The length represents the length of the payload. """ class SendMessage: r""" A Python class representing a message to be sent. - + This class wraps a Rust message meant for sending, facilitating the creation of such messages from Python and their subsequent use in Rust. """ - def __new__(cls, data:builtins.str | bytes) -> SendMessage: + def __new__(cls, data: builtins.str | bytes) -> SendMessage: r""" Constructs a new `SendMessage` instance from a string. - + This method allows for the creation of a `SendMessage` instance directly from Python using the provided string data. """ @@ -416,4 +505,3 @@ class TopicDetails: def messages_count(self) -> builtins.int: ... @property def partitions_count(self) -> builtins.int: ... - diff --git a/foreign/python/docker-compose.test.yml b/foreign/python/docker-compose.test.yml index 490aa5c744..92ffec6daf 100644 --- a/foreign/python/docker-compose.test.yml +++ b/foreign/python/docker-compose.test.yml @@ -21,8 +21,9 @@ services: iggy-server: build: context: ../.. - # Use debug build for faster compilation during testing - dockerfile: Dockerfile.debug + dockerfile: core/server/Dockerfile + args: + PROFILE: debug container_name: iggy-server-python-test networks: - python-test-network @@ -31,7 +32,7 @@ services: - "8080:8080" - "8090:8090" healthcheck: - test: ["CMD-SHELL", "timeout 5 bash -c ' IggyClient: @@ -67,7 +68,7 @@ async def iggy_client() -> IggyClient: def configure_asyncio(): """Configure asyncio settings for tests.""" # Set event loop policy if needed - if os.name == 'nt': # Windows + if os.name == "nt": # Windows asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) @@ -75,13 +76,9 @@ def configure_asyncio(): def pytest_configure(config): """Configure pytest with custom markers.""" config.addinivalue_line( - "markers", - "integration: marks tests as integration tests (may be slow)" - ) - config.addinivalue_line( - "markers", - "unit: marks tests as unit tests (fast)" + "markers", "integration: marks tests as integration tests (may be slow)" ) + config.addinivalue_line("markers", "unit: marks tests as unit tests (fast)") def pytest_collection_modifyitems(config, items): diff --git a/foreign/python/tests/test_iggy_sdk.py b/foreign/python/tests/test_iggy_sdk.py index 4cde470a08..604cd6bc27 100644 --- a/foreign/python/tests/test_iggy_sdk.py +++ b/foreign/python/tests/test_iggy_sdk.py @@ -23,16 +23,15 @@ """ import asyncio -from datetime import timedelta import uuid +from datetime import timedelta import pytest +from apache_iggy import AutoCommit, IggyClient, PollingStrategy, ReceiveMessage +from apache_iggy import SendMessage as Message from .utils import get_server_config, wait_for_ping, wait_for_server -from apache_iggy import IggyClient, PollingStrategy, AutoCommit, ReceiveMessage -from apache_iggy import SendMessage as Message - class TestConnectivity: """Test basic connectivity and authentication.""" @@ -69,7 +68,9 @@ def unique_stream_name(self): return f"test-stream-{uuid.uuid4().hex[:8]}" @pytest.mark.asyncio - async def test_create_and_get_stream(self, iggy_client: IggyClient, unique_stream_name): + async def test_create_and_get_stream( + self, iggy_client: IggyClient, unique_stream_name + ): """Test stream creation and retrieval.""" # Create stream await iggy_client.create_stream(unique_stream_name) @@ -102,24 +103,22 @@ def unique_names(self): """Generate unique stream and topic names.""" unique_id = uuid.uuid4().hex[:8] return { - 'stream': f"test-stream-{unique_id}", - 'topic': f"test-topic-{unique_id}" + "stream": f"test-stream-{unique_id}", + "topic": f"test-topic-{unique_id}", } @pytest.mark.asyncio async def test_create_and_get_topic(self, iggy_client: IggyClient, unique_names): """Test topic creation and retrieval.""" - stream_name = unique_names['stream'] - topic_name = unique_names['topic'] + stream_name = unique_names["stream"] + topic_name = unique_names["topic"] # Create stream first await iggy_client.create_stream(stream_name) # Create topic await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=2 + stream=stream_name, name=topic_name, partitions_count=2 ) # Get topic by name @@ -132,15 +131,13 @@ async def test_create_and_get_topic(self, iggy_client: IggyClient, unique_names) @pytest.mark.asyncio async def test_list_topics(self, iggy_client: IggyClient, unique_names): """Test listing topics in a stream.""" - stream_name = unique_names['stream'] - topic_name = unique_names['topic'] + stream_name = unique_names["stream"] + topic_name = unique_names["topic"] # Create stream and topic await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) # Get the topic we just created @@ -159,28 +156,24 @@ def message_setup(self): """Setup unique names and test data for messaging tests.""" unique_id = uuid.uuid4().hex[:8] return { - 'stream': f"msg-stream-{unique_id}", - 'topic': f"msg-topic-{unique_id}", - 'partition_id': 1, - 'messages': [ - f"Test message {i} - {unique_id}" for i in range(1, 4) - ] + "stream": f"msg-stream-{unique_id}", + "topic": f"msg-topic-{unique_id}", + "partition_id": 1, + "messages": [f"Test message {i} - {unique_id}" for i in range(1, 4)], } @pytest.mark.asyncio async def test_send_and_poll_messages(self, iggy_client: IggyClient, message_setup): """Test basic message sending and polling.""" - stream_name = message_setup['stream'] - topic_name = message_setup['topic'] - partition_id = message_setup['partition_id'] - test_messages = message_setup['messages'] + stream_name = message_setup["stream"] + topic_name = message_setup["topic"] + partition_id = message_setup["partition_id"] + test_messages = message_setup["messages"] # Setup stream and topic await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) # Send messages @@ -189,7 +182,7 @@ async def test_send_and_poll_messages(self, iggy_client: IggyClient, message_set stream=stream_name, topic=topic_name, partitioning=partition_id, - messages=messages + messages=messages, ) # Poll messages @@ -199,7 +192,7 @@ async def test_send_and_poll_messages(self, iggy_client: IggyClient, message_set partition_id=partition_id, polling_strategy=PollingStrategy.First(), count=10, - auto_commit=True + auto_commit=True, ) # Verify we got our messages @@ -212,19 +205,19 @@ async def test_send_and_poll_messages(self, iggy_client: IggyClient, message_set assert actual_payload == expected_msg @pytest.mark.asyncio - async def test_send_and_poll_messages_as_bytes(self, iggy_client: IggyClient, message_setup): + async def test_send_and_poll_messages_as_bytes( + self, iggy_client: IggyClient, message_setup + ): """Test basic message sending and polling with message payload as bytes.""" - stream_name = message_setup['stream'] - topic_name = message_setup['topic'] - partition_id = message_setup['partition_id'] - test_messages = message_setup['messages'] + stream_name = message_setup["stream"] + topic_name = message_setup["topic"] + partition_id = message_setup["partition_id"] + test_messages = message_setup["messages"] # Setup stream and topic await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) # Send messages @@ -233,7 +226,7 @@ async def test_send_and_poll_messages_as_bytes(self, iggy_client: IggyClient, me stream=stream_name, topic=topic_name, partitioning=partition_id, - messages=messages + messages=messages, ) # Poll messages @@ -243,7 +236,7 @@ async def test_send_and_poll_messages_as_bytes(self, iggy_client: IggyClient, me partition_id=partition_id, polling_strategy=PollingStrategy.First(), count=10, - auto_commit=True + auto_commit=True, ) # Verify we got our messages @@ -258,16 +251,14 @@ async def test_send_and_poll_messages_as_bytes(self, iggy_client: IggyClient, me @pytest.mark.asyncio async def test_message_properties(self, iggy_client: IggyClient, message_setup): """Test access to message properties.""" - stream_name = message_setup['stream'] - topic_name = message_setup['topic'] - partition_id = message_setup['partition_id'] + stream_name = message_setup["stream"] + topic_name = message_setup["topic"] + partition_id = message_setup["partition_id"] # Setup await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) # Send a test message @@ -277,7 +268,7 @@ async def test_message_properties(self, iggy_client: IggyClient, message_setup): stream=stream_name, topic=topic_name, partitioning=partition_id, - messages=[message] + messages=[message], ) # Poll and verify properties @@ -287,7 +278,7 @@ async def test_message_properties(self, iggy_client: IggyClient, message_setup): partition_id=partition_id, polling_strategy=PollingStrategy.Last(), count=1, - auto_commit=True + auto_commit=True, ) assert len(polled_messages) >= 1 @@ -310,26 +301,24 @@ def polling_setup(self): """Setup for polling strategy tests.""" unique_id = uuid.uuid4().hex[:8] return { - 'stream': f"poll-stream-{unique_id}", - 'topic': f"poll-topic-{unique_id}", - 'partition_id': 1, - 'messages': [f"Polling test {i} - {unique_id}" for i in range(5)] + "stream": f"poll-stream-{unique_id}", + "topic": f"poll-topic-{unique_id}", + "partition_id": 1, + "messages": [f"Polling test {i} - {unique_id}" for i in range(5)], } @pytest.mark.asyncio async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): """Test different polling strategies work correctly.""" - stream_name = polling_setup['stream'] - topic_name = polling_setup['topic'] - partition_id = polling_setup['partition_id'] - test_messages = polling_setup['messages'] + stream_name = polling_setup["stream"] + topic_name = polling_setup["topic"] + partition_id = polling_setup["partition_id"] + test_messages = polling_setup["messages"] # Setup await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) # Send test messages @@ -338,7 +327,7 @@ async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): stream=stream_name, topic=topic_name, partitioning=partition_id, - messages=messages + messages=messages, ) # Test First strategy @@ -348,7 +337,7 @@ async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): partition_id=partition_id, polling_strategy=PollingStrategy.First(), count=1, - auto_commit=False + auto_commit=False, ) assert len(first_messages) >= 1 @@ -359,7 +348,7 @@ async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): partition_id=partition_id, polling_strategy=PollingStrategy.Last(), count=1, - auto_commit=False + auto_commit=False, ) assert len(last_messages) >= 1 @@ -370,7 +359,7 @@ async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): partition_id=partition_id, polling_strategy=PollingStrategy.Next(), count=2, - auto_commit=False + auto_commit=False, ) assert len(next_messages) >= 1 @@ -380,9 +369,11 @@ async def test_polling_strategies(self, iggy_client: IggyClient, polling_setup): stream=stream_name, topic=topic_name, partition_id=partition_id, - polling_strategy=PollingStrategy.Offset(value=first_messages[0].offset()), + polling_strategy=PollingStrategy.Offset( + value=first_messages[0].offset() + ), count=1, - auto_commit=False + auto_commit=False, ) assert len(offset_messages) >= 1 @@ -421,11 +412,10 @@ async def test_create_topic_in_nonexistent_stream(self, iggy_client: IggyClient) with pytest.raises(RuntimeError): await iggy_client.create_topic( - stream=nonexistent_stream, - name=topic_name, - partitions_count=1 + stream=nonexistent_stream, name=topic_name, partitions_count=1 ) + class TestConsumerGroup: """Test consumer groups.""" @@ -434,27 +424,25 @@ def consumer_group_setup(self): """Setup for polling strategy tests.""" unique_id = uuid.uuid4().hex[:8] return { - 'consumer': f"consumer-group-consumer-{unique_id}", - 'stream': f"consumer-group-stream-{unique_id}", - 'topic': f"consumer-group-topic-{unique_id}", - 'partition_id': 1, - 'messages': [f"Consumer group test {i} - {unique_id}" for i in range(5)] + "consumer": f"consumer-group-consumer-{unique_id}", + "stream": f"consumer-group-stream-{unique_id}", + "topic": f"consumer-group-topic-{unique_id}", + "partition_id": 1, + "messages": [f"Consumer group test {i} - {unique_id}" for i in range(5)], } @pytest.mark.asyncio async def test_meta(self, iggy_client: IggyClient, consumer_group_setup): """Test that meta information can be read about the consumer group.""" - consumer_name = consumer_group_setup['consumer'] - stream_name = consumer_group_setup['stream'] - topic_name = consumer_group_setup['topic'] - partition_id = consumer_group_setup['partition_id'] + consumer_name = consumer_group_setup["consumer"] + stream_name = consumer_group_setup["stream"] + topic_name = consumer_group_setup["topic"] + partition_id = consumer_group_setup["partition_id"] # Setup await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) consumer = iggy_client.consumer_group( consumer_name, @@ -475,22 +463,22 @@ async def test_meta(self, iggy_client: IggyClient, consumer_group_setup): assert consumer.get_last_stored_offset(partition_id) is None @pytest.mark.asyncio - async def test_consume_messages(self, iggy_client: IggyClient, consumer_group_setup): + async def test_consume_messages( + self, iggy_client: IggyClient, consumer_group_setup + ): """Test that the consumer group can consume messages.""" - consumer_name = consumer_group_setup['consumer'] - stream_name = consumer_group_setup['stream'] - topic_name = consumer_group_setup['topic'] - partition_id = consumer_group_setup['partition_id'] - test_messages = consumer_group_setup['messages'] + consumer_name = consumer_group_setup["consumer"] + stream_name = consumer_group_setup["stream"] + topic_name = consumer_group_setup["topic"] + partition_id = consumer_group_setup["partition_id"] + test_messages = consumer_group_setup["messages"] # Setup received_messages = [] shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) consumer = iggy_client.consumer_group( @@ -509,9 +497,13 @@ async def take(message: ReceiveMessage) -> None: if len(received_messages) == 5: shutdown_event.set() - async def send() -> None: - await iggy_client.send_messages(stream_name, topic_name, partition_id, [Message(m) for m in test_messages]) + await iggy_client.send_messages( + stream_name, + topic_name, + partition_id, + [Message(m) for m in test_messages], + ) await asyncio.gather(consumer.consume_messages(take, shutdown_event), send()) @@ -520,18 +512,16 @@ async def send() -> None: @pytest.mark.asyncio async def test_shutdown(self, iggy_client: IggyClient, consumer_group_setup): """Test that the consumer group can be signaled to shutdown.""" - consumer_name = consumer_group_setup['consumer'] - stream_name = consumer_group_setup['stream'] - topic_name = consumer_group_setup['topic'] - partition_id = consumer_group_setup['partition_id'] + consumer_name = consumer_group_setup["consumer"] + stream_name = consumer_group_setup["stream"] + topic_name = consumer_group_setup["topic"] + partition_id = consumer_group_setup["partition_id"] # Setup shutdown_event = asyncio.Event() await iggy_client.create_stream(stream_name) await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1 + stream=stream_name, name=topic_name, partitions_count=1 ) consumer = iggy_client.consumer_group( diff --git a/scripts/check-backwards-compat.sh b/scripts/check-backwards-compat.sh new file mode 100755 index 0000000000..7f74677d98 --- /dev/null +++ b/scripts/check-backwards-compat.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. + +set -euo pipefail + +# ----------------------------- +# Config (overridable via args) +# ----------------------------- +MASTER_REF="${MASTER_REF:-master}" # branch or commit for "baseline" +PR_REF="${PR_REF:-HEAD}" # commit to test (assumes current checkout) +HOST="${HOST:-127.0.0.1}" +PORT="${PORT:-8090}" +WAIT_SECS="${WAIT_SECS:-60}" +BATCHES="${BATCHES:-50}" +MSGS_PER_BATCH="${MSGS_PER_BATCH:-100}" +KEEP_TMP="${KEEP_TMP:-false}" + +# ----------------------------- +# Helpers +# ----------------------------- +info(){ printf "\n\033[1;36m➀ %s\033[0m\n" "$*"; } +ok(){ printf "\033[0;32mβœ“ %s\033[0m\n" "$*"; } +err(){ printf "\033[0;31mβœ— %s\033[0m\n" "$*" >&2; } +die(){ err "$*"; exit 1; } + +need() { + command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1" +} + +wait_for_port() { + local host="$1" port="$2" deadline=$((SECONDS + WAIT_SECS)) + while (( SECONDS < deadline )); do + if command -v nc >/dev/null 2>&1; then + if nc -z "$host" "$port" 2>/dev/null; then return 0; fi + else + if (echo >"/dev/tcp/$host/$port") >/dev/null 2>&1; then return 0; fi + fi + sleep 1 + done + return 1 +} + +stop_pid() { + local pid="$1" name="${2:-process}" + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" || true + for _ in $(seq 1 15); do + kill -0 "$pid" 2>/dev/null || { ok "stopped $name (pid $pid)"; return 0; } + sleep 1 + done + err "$name (pid $pid) still running; sending SIGKILL" + kill -KILL "$pid" || true + fi +} + +print_logs_if_any() { + local dir="$1" + if compgen -G "$dir/local_data/logs/iggy*" > /dev/null; then + echo "---- $dir/local_data/logs ----" + cat "$dir"/local_data/logs/iggy* || true + echo "------------------------------" + else + echo "(no iggy logs found in $dir/local_data/logs)" + fi +} + +# ----------------------------- +# Args +# ----------------------------- +usage() { + cat </dev/null 2>&1 || true # optional, we'll use it if present + +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$REPO_ROOT" + +# Free the port proactively (best-effort) +pkill -f iggy-server >/dev/null 2>&1 || true + +TMP_ROOT="$(mktemp -d -t iggy-backcompat-XXXXXX)" +MASTER_DIR="$TMP_ROOT/master" +PR_DIR="$REPO_ROOT" # assume script is run from PR checkout +MASTER_LOG="$TMP_ROOT/server-master.stdout.log" +PR_LOG="$TMP_ROOT/server-pr.stdout.log" + +cleanup() { + # Stop any leftover iggy-server + pkill -f iggy-server >/dev/null 2>&1 || true + git worktree remove --force "$MASTER_DIR" >/dev/null 2>&1 || true + if [[ "$KEEP_TMP" != "true" ]]; then + rm -rf "$TMP_ROOT" || true + else + info "keeping temp dir: $TMP_ROOT" + fi +} +trap cleanup EXIT + +# ----------------------------- +# 1) Prepare master worktree +# ----------------------------- +info "Preparing baseline worktree at '$MASTER_REF'" +git fetch --all --tags --prune >/dev/null 2>&1 || true +git worktree add --force "$MASTER_DIR" "$MASTER_REF" +ok "worktree at $MASTER_DIR" + +# ----------------------------- +# 2) Build & run master server +# ----------------------------- +pushd "$MASTER_DIR" >/dev/null + +info "Building iggy-server & benches (baseline: $MASTER_REF)" +IGGY_CI_BUILD=true cargo build --bins +ok "built baseline" + +info "Starting iggy-server (baseline)" +set +e +( nohup target/debug/iggy-server >"$MASTER_LOG" 2>&1 & echo $! > "$TMP_ROOT/master.pid" ) +set -e +MASTER_PID="$(cat "$TMP_ROOT/master.pid")" +ok "iggy-server started (pid $MASTER_PID), logs: $MASTER_LOG" + +info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" +if ! wait_for_port "$HOST" "$PORT"; then + err "server did not become ready in ${WAIT_SECS}s" + print_logs_if_any "$MASTER_DIR" + [[ -f "$MASTER_LOG" ]] && tail -n 200 "$MASTER_LOG" || true + exit 1 +fi +ok "server is ready" + +# Producer bench (baseline) +info "Running producer bench on baseline" +BENCH_CMD=( target/debug/iggy-bench --verbose --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-producer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "producer bench done" + +# Consumer bench (baseline) +info "Running consumer bench on baseline" +BENCH_CMD=( target/debug/iggy-bench --verbose --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "consumer bench done (baseline)" + +# Stop baseline server +info "Stopping baseline server" +stop_pid "$MASTER_PID" "iggy-server(baseline)" +print_logs_if_any "$MASTER_DIR" + +# Clean baseline logs (like CI step) +if compgen -G "local_data/logs/iggy*" > /dev/null; then + rm -f local_data/logs/iggy* || true +fi + +# Snapshot local_data/ +info "Snapshotting baseline local_data/" +cp -a local_data "$TMP_ROOT/local_data" +ok "snapshot stored at $TMP_ROOT/local_data" + +popd >/dev/null + +# ----------------------------- +# 3) Build PR & restore data +# ----------------------------- +pushd "$PR_DIR" >/dev/null +info "Ensuring PR ref is present: $PR_REF" +git rev-parse --verify "$PR_REF^{commit}" >/dev/null 2>&1 || die "PR_REF '$PR_REF' not found" +git checkout -q "$PR_REF" + +info "Building iggy-server & benches (PR: $PR_REF)" +IGGY_CI_BUILD=true cargo build --bins +ok "built PR" + +info "Restoring baseline local_data/ into PR workspace" +rm -rf local_data +cp -a "$TMP_ROOT/local_data" ./local_data +ok "restored local_data/" + +# ----------------------------- +# 4) Run PR server & consumer bench +# ----------------------------- +info "Starting iggy-server (PR)" +set +e +( nohup target/debug/iggy-server >"$PR_LOG" 2>&1 & echo $! > "$TMP_ROOT/pr.pid" ) +set -e +PR_PID="$(cat "$TMP_ROOT/pr.pid")" +ok "iggy-server (PR) started (pid $PR_PID), logs: $PR_LOG" + +info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" +if ! wait_for_port "$HOST" "$PORT"; then + err "PR server did not become ready in ${WAIT_SECS}s" + print_logs_if_any "$PR_DIR" + [[ -f "$PR_LOG" ]] && tail -n 200 "$PR_LOG" || true + exit 1 +fi +ok "PR server is ready" + +# Only consumer bench against PR +info "Running consumer bench on PR (compat check)" +BENCH_CMD=( target/debug/iggy-bench --verbose --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "consumer bench done (PR)" + +# Stop PR server +info "Stopping PR server" +stop_pid "$PR_PID" "iggy-server(PR)" +print_logs_if_any "$PR_DIR" + +ok "backwards-compatibility check PASSED" +popd >/dev/null diff --git a/scripts/extract-version.sh b/scripts/extract-version.sh new file mode 100755 index 0000000000..46d5d2da7b --- /dev/null +++ b/scripts/extract-version.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Extract version information for Iggy components +# +# This script reads version information from various file formats based on +# the configuration in .github/config/publish.yml. It supports extracting +# versions from Cargo.toml, package.json, pyproject.toml, and other formats. +# +# Usage: +# ./extract-version.sh [--tag] [--go-sdk-version ] +# +# Examples: +# # Get version for Rust SDK +# ./extract-version.sh rust-sdk # Output: 0.7.0 +# +# # Get git tag for Rust SDK +# ./extract-version.sh rust-sdk --tag # Output: iggy-0.7.0 +# +# # Get version for Python SDK +# ./extract-version.sh sdk-python # Output: 0.5.0 +# +# # Get tag for Node SDK +# ./extract-version.sh sdk-node --tag # Output: node-sdk-0.5.0 +# +# # Get version for Go SDK (requires explicit version) +# ./extract-version.sh sdk-go --go-sdk-version 1.2.3 # Output: 1.2.3 +# +# The script uses the configuration from .github/config/publish.yml to determine: +# - Where to find the version file (version_file) +# - What regex pattern to use for extraction (version_regex) +# - How to format the git tag (tag_pattern) +# - Package name for Rust crates (package) + +set -euo pipefail + +# Check for required tools +if ! command -v yq &> /dev/null; then + echo "Error: yq is required but not installed" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CONFIG_FILE="$REPO_ROOT/.github/config/publish.yml" + +# Parse arguments +COMPONENT="${1:-}" +RETURN_TAG=false +GO_SDK_VERSION="" + +shift || true +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) + RETURN_TAG=true + shift + ;; + --go-sdk-version) + GO_SDK_VERSION="${2:-}" + shift 2 || shift + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$COMPONENT" ]]; then + echo "Usage: $0 [--tag] [--go-sdk-version ]" >&2 + echo "" >&2 + echo "Available components:" >&2 + yq eval '.components | keys | .[]' "$CONFIG_FILE" | sed 's/^/ - /' >&2 + exit 1 +fi + +# Check if component exists +if ! yq eval ".components.\"$COMPONENT\"" "$CONFIG_FILE" | grep -q .; then + echo "Error: Unknown component '$COMPONENT'" >&2 + echo "" >&2 + echo "Available components:" >&2 + yq eval '.components | keys | .[]' "$CONFIG_FILE" | sed 's/^/ - /' >&2 + exit 1 +fi + +# Extract component configuration +get_config() { + local key="$1" + yq eval ".components.\"$COMPONENT\".$key // \"\"" "$CONFIG_FILE" +} + +# Generic regex-based extraction +extract_version_with_regex() { + local file="$1" + local regex="$2" + + if [[ ! -f "$REPO_ROOT/$file" ]]; then + echo "Error: File not found: $file" >&2 + return 1 + fi + + # Special handling for XML files (C# .csproj) + if [[ "$file" == *.csproj ]] || [[ "$file" == *.xml ]]; then + # Extract version from XML tags like or + grep -E '<(PackageVersion|Version)>' "$REPO_ROOT/$file" | head -1 | sed -E 's/.*<[^>]+>([^<]+)<.*/\1/' | tr -d ' ' + elif command -v perl &> /dev/null; then + # Use perl for more powerful regex support (supports multiline and lookarounds) + # Use m{} instead of // to avoid issues with slashes in regex + perl -0777 -ne "if (m{$regex}) { print \$1; exit; }" "$REPO_ROOT/$file" + else + # Fallback to grep -P if available + if grep -P "" /dev/null 2>/dev/null; then + grep -Pzo "$regex" "$REPO_ROOT/$file" | grep -Pao '[0-9]+\.[0-9]+\.[0-9]+[^"]*' | head -1 + else + # Basic fallback - may not work for all patterns + grep -E "$regex" "$REPO_ROOT/$file" | head -1 | sed -E "s/.*$regex.*/\1/" + fi + fi +} + +# Extract version using cargo metadata (for Rust packages) +extract_cargo_version() { + local package="$1" + local cargo_file="$2" + + cd "$REPO_ROOT" + + # Try cargo metadata first (most reliable) + if command -v cargo &> /dev/null && command -v jq &> /dev/null; then + version=$(cargo metadata --no-deps --format-version=1 2>/dev/null | \ + jq -r --arg pkg "$package" '.packages[] | select(.name == $pkg) | .version' | \ + head -1) + + if [[ -n "$version" ]]; then + echo "$version" + return 0 + fi + fi + + # Fallback to direct Cargo.toml parsing using the regex from config + local version_regex + version_regex=$(get_config "version_regex") + if [[ -n "$version_regex" && -f "$REPO_ROOT/$cargo_file" ]]; then + extract_version_with_regex "$cargo_file" "$version_regex" + fi +} + +# Main version extraction logic +VERSION="" +VERSION_FILE=$(get_config "version_file") +VERSION_REGEX=$(get_config "version_regex") +PACKAGE=$(get_config "package") + +# Special handling for Go SDK (version must be provided) +if [[ "$COMPONENT" == "sdk-go" ]]; then + VERSION="$GO_SDK_VERSION" + if [[ -z "$VERSION" ]]; then + echo "Error: Go version must be provided with --go-version flag" >&2 + exit 1 + fi +# For Rust components with cargo metadata support +elif [[ "$COMPONENT" == rust-* ]] && [[ -n "$PACKAGE" ]]; then + # Use package name from config if available + VERSION=$(extract_cargo_version "$PACKAGE" "$VERSION_FILE") + + # Fallback to regex-based extraction if cargo metadata failed + if [[ -z "$VERSION" ]] && [[ -n "$VERSION_FILE" ]] && [[ -n "$VERSION_REGEX" ]]; then + VERSION=$(extract_version_with_regex "$VERSION_FILE" "$VERSION_REGEX") + fi +# Generic extraction using version_file and version_regex +elif [[ -n "$VERSION_FILE" ]] && [[ -n "$VERSION_REGEX" ]]; then + VERSION=$(extract_version_with_regex "$VERSION_FILE" "$VERSION_REGEX") +else + echo "Error: No version extraction method available for component '$COMPONENT'" >&2 + exit 1 +fi + +# Validate version was found +if [[ -z "$VERSION" ]]; then + echo "Error: Could not extract version for component '$COMPONENT'" >&2 + if [[ -n "$VERSION_FILE" ]]; then + echo " Checked file: $VERSION_FILE" >&2 + fi + if [[ -n "$VERSION_REGEX" ]]; then + echo " Using regex: $VERSION_REGEX" >&2 + fi + exit 1 +fi + +# Return tag or version based on flag +if [[ "$RETURN_TAG" == "true" ]]; then + TAG_PATTERN=$(get_config "tag_pattern") + if [[ -z "$TAG_PATTERN" ]]; then + echo "Error: No tag pattern defined for component '$COMPONENT'" >&2 + exit 1 + fi + + # Replace the capture group in the pattern with the actual version + # The pattern has a capture group like "^iggy-([0-9]+\\.[0-9]+\\.[0-9]+...)$" + # We need to replace the (...) part with the actual version + + # Extract the prefix (everything before the first capture group) + PREFIX=$(echo "$TAG_PATTERN" | sed -E 's/^(\^?)([^(]*)\(.*/\2/') + + # Extract the suffix (everything after the capture group) + SUFFIX=$(echo "$TAG_PATTERN" | sed -E 's/.*\)[^)]*(\$?)$/\1/') + + # Build the tag + TAG="${PREFIX}${VERSION}${SUFFIX}" + + # Remove regex anchors if present + TAG=$(echo "$TAG" | sed 's/^\^//; s/\$$//') + + echo "$TAG" +else + echo "$VERSION" +fi \ No newline at end of file diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh index fe20972624..45d334c9ab 100755 --- a/scripts/run-bdd-tests.sh +++ b/scripts/run-bdd-tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,75 +16,56 @@ # specific language governing permissions and limitations # under the License. -set -e +set -Eeuo pipefail SDK=${1:-"all"} FEATURE=${2:-"scenarios/basic_messaging.feature"} -echo "πŸ§ͺ Running BDD tests for SDK: $SDK" -echo "πŸ“ Feature file: $FEATURE" +export DOCKER_BUILDKIT=1 FEATURE -# Change to BDD directory cd "$(dirname "$0")/../bdd" -case $SDK in -"rust") - echo "πŸ¦€ Running Rust BDD tests..." - docker compose build --no-cache iggy-server rust-bdd - docker compose up --abort-on-container-exit rust-bdd - ;; -"python") - echo "🐍 Running Python BDD tests..." - docker compose build --no-cache iggy-server python-bdd - docker compose up --abort-on-container-exit python-bdd - ;; -"go") - echo "🐹 Running Go BDD tests..." - docker compose build --no-cache iggy-server go-bdd - docker compose up --abort-on-container-exit go-bdd - ;; -"node") - echo "πŸ’πŸš€ Running node BDD tests..." - docker compose build --no-cache iggy-server node-bdd - docker compose up --abort-on-container-exit node-bdd - ;; -"csharp") - echo "πŸ”· Running csharp BDD tests..." - docker compose build --no-cache iggy-server csharp-bdd - docker compose up --abort-on-container-exit csharp-bdd - ;; -"all") - echo "πŸš€ Running all SDK BDD tests..." - echo "πŸ¦€ Starting with Rust tests..." - #docker compose build --no-cache iggy-server rust-bdd python-bdd go-bdd node-bdd csharp-bdd - docker compose up --abort-on-container-exit rust-bdd - echo "🐍 Now running Python tests..." - docker compose up --abort-on-container-exit python-bdd - echo "🐹 Now running Go tests..." - docker compose up --abort-on-container-exit go-bdd - echo "πŸ’πŸš€ Now running node BDD tests..." - docker compose up --abort-on-container-exit node-bdd - echo "πŸ”· Now running csharp BDD tests..." - docker compose up --abort-on-container-exit csharp-bdd - ;; -"clean") - echo "🧹 Cleaning up Docker resources..." - docker compose down -v - docker compose rm -f - ;; -*) - echo "❌ Unknown SDK: $SDK" - echo "πŸ“– Usage: $0 [rust|python|go|all|clean] [feature_file]" - echo "πŸ“– Examples:" - echo " $0 rust # Run Rust tests only" - echo " $0 python # Run Python tests only" - echo " $0 go # Run Go tests only" - echo " $0 node # Run Node.js tests only" - echo " $0 csharp # Run csharp tests only" - echo " $0 all # Run all SDK tests" - echo " $0 clean # Clean up Docker resources" - exit 1 - ;; +log(){ printf "%b\n" "$*"; } + +cleanup(){ + log "🧹 cleaning up containers & volumes…" + docker compose down -v --remove-orphans >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +log "πŸ§ͺ Running BDD tests for SDK: ${SDK}" +log "πŸ“ Feature file: ${FEATURE}" + +run_suite(){ + local svc="$1" emoji="$2" label="$3" + log "${emoji} ${label}…" + set +e + docker compose up --build --abort-on-container-exit --exit-code-from "$svc" "$svc" + local code=$? + set -e + docker compose down -v --remove-orphans >/dev/null 2>&1 || true + return "$code" +} + +case "$SDK" in + rust) run_suite rust-bdd "πŸ¦€" "Running Rust BDD tests" ;; + python) run_suite python-bdd "🐍" "Running Python BDD tests" ;; + go) run_suite go-bdd "🐹" "Running Go BDD tests" ;; + node) run_suite node-bdd "πŸ’πŸš€" "Running Node BDD tests" ;; + csharp) run_suite csharp-bdd "πŸ”·" "Running C# BDD tests" ;; + all) + run_suite rust-bdd "πŸ¦€" "Running Rust BDD tests" || exit $? + run_suite python-bdd "🐍" "Running Python BDD tests" || exit $? + run_suite go-bdd "🐹" "Running Go BDD tests" || exit $? + run_suite node-bdd "πŸ’πŸš€" "Running Node BDD tests" || exit $? + run_suite csharp-bdd "πŸ”·" "Running C# BDD tests" || exit $? + ;; + clean) + cleanup; exit 0 ;; + *) + log "❌ Unknown SDK: ${SDK}" + log "πŸ“– Usage: $0 [rust|python|go|node|csharp|all|clean] [feature_file]" + exit 2 ;; esac -echo "βœ… BDD tests completed for: $SDK" +log "βœ… BDD tests completed for: ${SDK}" diff --git a/scripts/run-go-examples-from-readme.sh b/scripts/run-go-examples-from-readme.sh index 1bbdd478e0..aca887be0e 100755 --- a/scripts/run-go-examples-from-readme.sh +++ b/scripts/run-go-examples-from-readme.sh @@ -86,20 +86,30 @@ test -d local_data && rm -fr local_data test -e ${LOG_FILE} && rm ${LOG_FILE} test -e ${PID_FILE} && rm ${PID_FILE} -# Build binaries -echo "Building binaries..." +# Check if server binary exists +SERVER_BIN="" if [ -n "${TARGET}" ]; then - cargo build --target "${TARGET}" + SERVER_BIN="target/${TARGET}/debug/iggy-server" else - cargo build + SERVER_BIN="target/debug/iggy-server" fi -# Run iggy server and let it run in the background -if [ -n "${TARGET}" ]; then - cargo run --target "${TARGET}" --bin iggy-server &>${LOG_FILE} & -else - cargo run --bin iggy-server &>${LOG_FILE} & +if [ ! -f "${SERVER_BIN}" ]; then + echo "Error: Server binary not found at ${SERVER_BIN}" + echo "Please build the server binary before running this script:" + if [ -n "${TARGET}" ]; then + echo " cargo build --target ${TARGET} --bin iggy-server" + else + echo " cargo build --bin iggy-server" + fi + exit 1 fi + +echo "Using server binary at ${SERVER_BIN}" + +# Run iggy server using the prebuilt binary +echo "Starting server from ${SERVER_BIN}..." +${SERVER_BIN} &>${LOG_FILE} & echo $! >${PID_FILE} # Wait until "Iggy server has started" string is present inside iggy-server.log diff --git a/scripts/run-rust-examples-from-readme.sh b/scripts/run-rust-examples-from-readme.sh index ddd4bcec72..ed2e5617d2 100755 --- a/scripts/run-rust-examples-from-readme.sh +++ b/scripts/run-rust-examples-from-readme.sh @@ -55,20 +55,51 @@ test -d local_data && rm -fr local_data test -e ${LOG_FILE} && rm ${LOG_FILE} test -e ${PID_FILE} && rm ${PID_FILE} -# Build binaries -echo "Building binaries..." +# Check if server binary exists +SERVER_BIN="" if [ -n "${TARGET}" ]; then - cargo build --target "${TARGET}" + SERVER_BIN="target/${TARGET}/debug/iggy-server" else - cargo build + SERVER_BIN="target/debug/iggy-server" fi -# Run iggy server and let it run in the background +if [ ! -f "${SERVER_BIN}" ]; then + echo "Error: Server binary not found at ${SERVER_BIN}" + echo "Please build the server binary before running this script:" + if [ -n "${TARGET}" ]; then + echo " cargo build --target ${TARGET} --bin iggy-server" + else + echo " cargo build --bin iggy-server" + fi + exit 1 +fi + +echo "Using server binary at ${SERVER_BIN}" + +# Check that CLI and examples are built +CLI_BIN="" if [ -n "${TARGET}" ]; then - cargo run --target "${TARGET}" --bin iggy-server &>${LOG_FILE} & + CLI_BIN="target/${TARGET}/debug/iggy" else - cargo run --bin iggy-server &>${LOG_FILE} & + CLI_BIN="target/debug/iggy" fi + +if [ ! -f "${CLI_BIN}" ]; then + echo "Error: CLI binary not found at ${CLI_BIN}" + echo "Please build the CLI and examples before running this script:" + if [ -n "${TARGET}" ]; then + echo " cargo build --target ${TARGET} --bin iggy --examples" + else + echo " cargo build --bin iggy --examples" + fi + exit 1 +fi + +echo "Using CLI binary at ${CLI_BIN}" + +# Run iggy server using the prebuilt binary +echo "Starting server from ${SERVER_BIN}..." +${SERVER_BIN} &>${LOG_FILE} & echo $! >${PID_FILE} # Wait until "Iggy server has started" string is present inside iggy-server.log diff --git a/scripts/sync-rust-version.sh b/scripts/sync-rust-version.sh index 09ebe2e672..ccaa8012f9 100755 --- a/scripts/sync-rust-version.sh +++ b/scripts/sync-rust-version.sh @@ -16,27 +16,131 @@ # specific language governing permissions and limitations # under the License. +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Default mode +MODE="" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --check) + MODE="check" + shift + ;; + --fix) + MODE="fix" + shift + ;; + --help|-h) + echo "Usage: $0 [--check|--fix]" + echo "" + echo "Sync Rust version from rust-toolchain.toml to all Dockerfiles" + echo "" + echo "Options:" + echo " --check Check if all Dockerfiles have the correct Rust version" + echo " --fix Update all Dockerfiles to use the correct Rust version" + echo " --help Show this help message" + exit 0 + ;; + *) + echo -e "${RED}Error: Unknown option $1${NC}" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Require mode to be specified +if [ -z "$MODE" ]; then + echo -e "${RED}Error: Please specify either --check or --fix${NC}" + echo "Use --help for usage information" + exit 1 +fi + +# Get the repository root (parent of scripts directory) +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" # Extract Rust version from rust-toolchain.toml RUST_VERSION=$(grep 'channel' rust-toolchain.toml | sed 's/.*"\(.*\)".*/\1/') if [ -z "$RUST_VERSION" ]; then - echo "Error: Could not extract Rust version from rust-toolchain.toml" + echo -e "${RED}Error: Could not extract Rust version from rust-toolchain.toml${NC}" exit 1 fi # Strip trailing ".0" -> e.g., 1.89.0 -> 1.89 (no change if it doesn't end in .0) -RUST_TAG=$(echo "$RUST_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+)\.0$/\1.\2/') +RUST_VERSION_SHORT=$(echo "$RUST_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+)\.0$/\1.\2/') + +echo "Rust version from rust-toolchain.toml: ${GREEN}$RUST_VERSION${NC} (using ${GREEN}$RUST_VERSION_SHORT${NC} for Dockerfiles)" +echo "" + +# Find all Dockerfiles +DOCKERFILES=$(find . -name "Dockerfile*" -type f | grep -v node_modules | grep -v target | sort) + +# Track misaligned files +MISALIGNED_FILES=() +TOTAL_FILES=0 +FIXED_FILES=0 + +for dockerfile in $DOCKERFILES; do + # Skip files without ARG RUST_VERSION + if ! grep -q "^ARG RUST_VERSION=" "$dockerfile" 2>/dev/null; then + continue + fi + + TOTAL_FILES=$((TOTAL_FILES + 1)) -echo "Syncing Rust version $RUST_VERSION (using tag: $RUST_TAG) to Dockerfiles..." + # Get current version in the Dockerfile + CURRENT_VERSION=$(grep "^ARG RUST_VERSION=" "$dockerfile" | head -1 | sed 's/^ARG RUST_VERSION=//') -# Update regular rust image (no suffix) -# Matches things like: FROM rust:1.88, FROM rust:1.88.1, etc. -sed -Ei "s|(FROM[[:space:]]+rust:)[0-9]+(\.[0-9]+){1,2}|\1$RUST_TAG|g" bdd/rust/Dockerfile + if [ "$MODE" = "check" ]; then + if [ "$CURRENT_VERSION" != "$RUST_VERSION_SHORT" ]; then + MISALIGNED_FILES+=("$dockerfile") + echo -e "${RED}βœ—${NC} $dockerfile: ${RED}$CURRENT_VERSION${NC} (expected: ${GREEN}$RUST_VERSION_SHORT${NC})" + else + echo -e "${GREEN}βœ“${NC} $dockerfile: $CURRENT_VERSION" + fi + elif [ "$MODE" = "fix" ]; then + if [ "$CURRENT_VERSION" != "$RUST_VERSION_SHORT" ]; then + # Update the ARG RUST_VERSION line + sed -i "s/^ARG RUST_VERSION=.*/ARG RUST_VERSION=$RUST_VERSION_SHORT/" "$dockerfile" + FIXED_FILES=$((FIXED_FILES + 1)) + echo -e "${GREEN}Fixed${NC} $dockerfile: ${RED}$CURRENT_VERSION${NC} -> ${GREEN}$RUST_VERSION_SHORT${NC}" + else + echo -e "${GREEN}βœ“${NC} $dockerfile: already correct ($RUST_VERSION_SHORT)" + fi + fi +done -# Update slim-bookworm image -sed -Ei "s|(FROM[[:space:]]+rust:)[0-9]+(\.[0-9]+){1,2}-slim-bookworm|\1$RUST_TAG-slim-bookworm|g" core/bench/dashboard/server/Dockerfile +echo "" +echo "────────────────────────────────────────────────" -echo "Updated Dockerfiles to use:" -echo " - Regular image: rust:$RUST_TAG" -echo " - Slim bookworm: rust:$RUST_TAG-slim-bookworm" +if [ "$MODE" = "check" ]; then + if [ ${#MISALIGNED_FILES[@]} -eq 0 ]; then + echo -e "${GREEN}βœ“ All $TOTAL_FILES Dockerfiles are aligned with Rust version $RUST_VERSION_SHORT${NC}" + exit 0 + else + echo -e "${RED}βœ— Found ${#MISALIGNED_FILES[@]} misaligned Dockerfile(s) out of $TOTAL_FILES:${NC}" + for file in "${MISALIGNED_FILES[@]}"; do + echo -e " ${RED}β€’ $file${NC}" + done + echo "" + echo -e "${YELLOW}Run '$0 --fix' to fix these files${NC}" + exit 1 + fi +elif [ "$MODE" = "fix" ]; then + if [ $FIXED_FILES -eq 0 ]; then + echo -e "${GREEN}βœ“ All $TOTAL_FILES Dockerfiles were already aligned with Rust version $RUST_VERSION_SHORT${NC}" + else + echo -e "${GREEN}βœ“ Fixed $FIXED_FILES out of $TOTAL_FILES Dockerfiles to use Rust version $RUST_VERSION_SHORT${NC}" + fi + exit 0 +fi \ No newline at end of file diff --git a/web/Dockerfile b/web/Dockerfile index 846f397fce..e234b44fe2 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -16,6 +16,8 @@ # under the License. FROM node:lts-alpine AS base +ARG IGGY_CI_BUILD +ENV IGGY_CI_BUILD=${IGGY_CI_BUILD} ENV NPM_CONFIG_LOGLEVEL=warn ENV NPM_CONFIG_COLOR=false WORKDIR /home/node/app @@ -30,4 +32,4 @@ WORKDIR /home/node/app COPY --chown=node:node --from=development /home/node/app/node_modules /home/node/app/node_modules RUN npm run build EXPOSE 3050 -CMD ["npm", "run", "preview"] \ No newline at end of file +CMD ["npm", "run", "preview"]