From bb95f055401cd781ba2a86fb21f3eb758eaa006e Mon Sep 17 00:00:00 2001 From: Chiamaka Date: Wed, 16 Jul 2025 12:50:30 +0000 Subject: [PATCH 1/3] optimize: reduce Docker build time --- .github/workflows/publish-standalone.yml | 59 ++++++++----- .github/workflows/standalone-build.yml | 104 +++++++++++++++++++++++ Dockerfile.standalone | 42 +++++---- 3 files changed, 170 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/standalone-build.yml diff --git a/.github/workflows/publish-standalone.yml b/.github/workflows/publish-standalone.yml index 8e052b63c..4e21ff041 100644 --- a/.github/workflows/publish-standalone.yml +++ b/.github/workflows/publish-standalone.yml @@ -1,40 +1,59 @@ -name: Build and Publish Standalone Docker Image - +name: Build and Publish Stakgraph Standalone on: - release: - types: [published] + push: + branches: + - main + paths: + - 'standalone/**' + - 'ast/**' + - 'lsp/**' + - 'Dockerfile.standalone' + - '.github/workflows/standalone-build.yml' jobs: - build-and-push: + build-and-publish: runs-on: ubuntu-latest - permissions: - contents: read - packages: write - + name: Build and Publish Docker Image steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry + + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image + + # Setup cache + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + # Build and push the image + - name: Build and Push Docker Image uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.standalone - platforms: linux/amd64,linux/arm64 + file: ./Dockerfile.standalone push: true - # Use registry cache which is more reliable for multi-platform - cache-from: type=registry,ref=ghcr.io/stakwork/stakgraph-standalone:buildcache - cache-to: type=registry,ref=ghcr.io/stakwork/stakgraph-standalone:buildcache,mode=max tags: | - ghcr.io/stakwork/stakgraph-standalone:${{ github.ref_name }} ghcr.io/stakwork/stakgraph-standalone:latest + ghcr.io/stakwork/stakgraph-standalone:${{ github.sha }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + platforms: linux/amd64 + build-args: | + BUILDKIT_INLINE_CACHE=1 + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file diff --git a/.github/workflows/standalone-build.yml b/.github/workflows/standalone-build.yml new file mode 100644 index 000000000..75ff1d7b7 --- /dev/null +++ b/.github/workflows/standalone-build.yml @@ -0,0 +1,104 @@ +name: Stakgraph Standalone Test +on: + pull_request: + branches: + - main + +jobs: + run-standalone-test: + runs-on: ubuntu-latest + name: Run standalone tests + steps: + - name: Enable docker.host.internal for Ubuntu + run: | + pwd && sudo bash -c 'echo "172.17.0.1 host.docker.internal" >> /etc/hosts' + + - name: Checkout from Github + uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo Dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + # Set up Docker cache + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Pull Neo4j Docker image + run: docker pull sphinxlightning/sphinx-neo4j:latest + + - name: Give Permission + working-directory: ./mcp + run: | + mkdir -p ./.neo4j/plugins + sudo chown -R 1000:1000 ./.neo4j/plugins + sudo chmod -R 777 ./.neo4j/plugins + sudo chmod -R 777 ./conf + + - name: Run Neo4j + working-directory: ./mcp + run: docker compose -f neo4j.yaml up -d + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.24.1" + cache: true + + - name: Install gopls + run: go install golang.org/x/tools/gopls@latest + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: 'npm' + + - name: Install TypeScript and TypeScript LSP + run: npm install -g typescript typescript-language-server + + - name: Wait for Neo4j to be Ready + run: | + echo "Waiting for Neo4j to be healthy..." + timeout 300s sh -c ' + until docker inspect --format "{{json .State.Health.Status }}" neo4j.sphinx | grep -q "healthy"; do + echo "Neo4j is not ready yet..." + sleep 5 + done' + echo "Neo4j is healthy!" + + # Run all tests in parallel with a shared database + - name: Run All Tests + run: | + export USE_LSP=false + export LSP_SKIP_POST_CLONE=1 + export TEST_REF_ID="test_ref_id" + cargo test --features neo4j -- --test-threads=1 + + - name: Cleanup + if: always() + run: | + if [ -f standalone.pid ]; then + kill $(cat standalone.pid) || true + fi + docker compose -f ./mcp/neo4j.yaml down + + diff --git a/Dockerfile.standalone b/Dockerfile.standalone index 09d1e91a0..55b84e80f 100644 --- a/Dockerfile.standalone +++ b/Dockerfile.standalone @@ -1,36 +1,48 @@ -FROM rust:1.87.0 AS builder - -WORKDIR /app - +FROM rust:1.87.0 AS planner +WORKDIR /plan # Create a temporary workspace with only the 3 crates we need RUN echo '[workspace]' > Cargo.toml && \ echo 'members = ["ast", "lsp", "standalone"]' >> Cargo.toml && \ echo 'resolver = "2"' >> Cargo.toml -# Copy ONLY the Cargo.toml files for our 3 crates +# Copy only files needed for dependency calculations to create a stable dependency layer COPY ast/Cargo.toml ast/Cargo.toml COPY lsp/Cargo.toml lsp/Cargo.toml COPY standalone/Cargo.toml standalone/Cargo.toml +COPY Cargo.lock . -# Create dummy src files for all 3 crates +# Create a dummy project structure (minimal files needed for cargo to calculate dependencies) RUN mkdir -p ast/src lsp/src standalone/src && \ - echo "fn main() {}" > ast/src/main.rs && \ - echo "fn main() {}" > lsp/src/main.rs && \ - echo "fn main() {}" > standalone/src/main.rs + echo 'fn main() {}' > ast/src/lib.rs && \ + echo 'fn main() {}' > lsp/src/lib.rs && \ + echo 'fn main() {}' > standalone/src/main.rs -# Build dependencies for standalone (this will pull in ast/lsp as workspace deps) -RUN cargo build --release --bin standalone --features neo4j +FROM rust:1.87.0 AS cacher +WORKDIR /app +COPY --from=planner /plan . + +# Build dependencies only (leveraging cargo's dependency caching) +RUN cargo build --release --bin standalone --features neo4j && \ + rm -rf ast/src lsp/src standalone/src + +FROM rust:1.87.0 AS builder +WORKDIR /app -# Remove dummy files and temp workspace -RUN rm -rf ast/src lsp/src standalone/src +COPY --from=cacher /app/target target +COPY --from=cacher /usr/local/cargo /usr/local/cargo +COPY --from=cacher /app/Cargo.toml . +COPY --from=cacher /app/Cargo.lock . +COPY --from=cacher /app/ast/Cargo.toml ./ast/ +COPY --from=cacher /app/lsp/Cargo.toml ./lsp/ +COPY --from=cacher /app/standalone/Cargo.toml ./standalone/ -# Copy actual source code +# Now copy the actual source code COPY ast/src ast/src COPY lsp/src lsp/src COPY standalone/src standalone/src COPY standalone/static standalone/static -# Build the actual application (only this rebuilds when source changes) +# Build the application with the actual source (much faster now with all dependencies pre-built) RUN cargo build --release --bin standalone --features neo4j FROM sphinxlightning/stakgraph-lsp:latest From e5a1a65e31930537ce713ba90399f03b86026996 Mon Sep 17 00:00:00 2001 From: Chiamaka Date: Fri, 18 Jul 2025 00:52:14 +0000 Subject: [PATCH 2/3] clean code --- .github/workflows/standalone-build.yml | 104 ------------------------- 1 file changed, 104 deletions(-) delete mode 100644 .github/workflows/standalone-build.yml diff --git a/.github/workflows/standalone-build.yml b/.github/workflows/standalone-build.yml deleted file mode 100644 index 75ff1d7b7..000000000 --- a/.github/workflows/standalone-build.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Stakgraph Standalone Test -on: - pull_request: - branches: - - main - -jobs: - run-standalone-test: - runs-on: ubuntu-latest - name: Run standalone tests - steps: - - name: Enable docker.host.internal for Ubuntu - run: | - pwd && sudo bash -c 'echo "172.17.0.1 host.docker.internal" >> /etc/hosts' - - - name: Checkout from Github - uses: actions/checkout@v4 - - - name: Set up Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo Dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - # Set up Docker cache - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Pull Neo4j Docker image - run: docker pull sphinxlightning/sphinx-neo4j:latest - - - name: Give Permission - working-directory: ./mcp - run: | - mkdir -p ./.neo4j/plugins - sudo chown -R 1000:1000 ./.neo4j/plugins - sudo chmod -R 777 ./.neo4j/plugins - sudo chmod -R 777 ./conf - - - name: Run Neo4j - working-directory: ./mcp - run: docker compose -f neo4j.yaml up -d - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: "1.24.1" - cache: true - - - name: Install gopls - run: go install golang.org/x/tools/gopls@latest - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: 'npm' - - - name: Install TypeScript and TypeScript LSP - run: npm install -g typescript typescript-language-server - - - name: Wait for Neo4j to be Ready - run: | - echo "Waiting for Neo4j to be healthy..." - timeout 300s sh -c ' - until docker inspect --format "{{json .State.Health.Status }}" neo4j.sphinx | grep -q "healthy"; do - echo "Neo4j is not ready yet..." - sleep 5 - done' - echo "Neo4j is healthy!" - - # Run all tests in parallel with a shared database - - name: Run All Tests - run: | - export USE_LSP=false - export LSP_SKIP_POST_CLONE=1 - export TEST_REF_ID="test_ref_id" - cargo test --features neo4j -- --test-threads=1 - - - name: Cleanup - if: always() - run: | - if [ -f standalone.pid ]; then - kill $(cat standalone.pid) || true - fi - docker compose -f ./mcp/neo4j.yaml down - - From 48d699329a9fda05359789c283385b13bb68eb40 Mon Sep 17 00:00:00 2001 From: Chiamaka Date: Sat, 19 Jul 2025 05:41:01 +0000 Subject: [PATCH 3/3] feat(ci): optimize Docker build with multi-platform support (ARM64/AMD64) --- .github/workflows/publish-standalone.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish-standalone.yml b/.github/workflows/publish-standalone.yml index 4e21ff041..db3e9b1a2 100644 --- a/.github/workflows/publish-standalone.yml +++ b/.github/workflows/publish-standalone.yml @@ -4,11 +4,10 @@ on: branches: - main paths: - - 'standalone/**' - - 'ast/**' - - 'lsp/**' - - 'Dockerfile.standalone' - - '.github/workflows/standalone-build.yml' + - "standalone/**" + - "ast/**" + - "lsp/**" + - "Dockerfile.standalone" jobs: build-and-publish: @@ -17,17 +16,17 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + # Setup cache - name: Cache Docker layers uses: actions/cache@v3 @@ -36,7 +35,7 @@ jobs: key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - + # Build and push the image - name: Build and Push Docker Image uses: docker/build-push-action@v5 @@ -49,11 +48,10 @@ jobs: ghcr.io/stakwork/stakgraph-standalone:${{ github.sha }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 build-args: | BUILDKIT_INLINE_CACHE=1 - - name: Move cache run: | rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file + mv /tmp/.buildx-cache-new /tmp/.buildx-cache