cuda_129_python_dev.yml

name: CE-CUDA-129-PYTHON-develop

on:
  schedule:
    - cron: '0 19 * * *'
  workflow_dispatch: # 可选：允许手动点 Run
    inputs:
      use_release:
        description: '是否使用预编译的develop版本'
        type: choice
        options:
          - 'true'
          - 'false'
        default: 'true'
        required: false
permissions: read-all

concurrency:
  group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  PR_ID: ${{ github.event.pull_request.number || '0' }}
  COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  ci_scripts: /paddle/ci
  BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
  CI_name: test
  no_proxy: "bcebos.com,.bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
  docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-coverage-test"
  paddle_url: "https://paddle-qa.bj.bcebos.com/paddle-pipeline/Develop-TagBuild-Training-Linux-Gpu-Cuda12.9-Cudnn9.9-Trt10.5-Mkl-Avx-Gcc11-SelfBuiltPypiUse/latest/"
  use_release: ${{ inputs.use_release || 'true' }}
defaults:
  run:
    shell: bash

jobs:
  single_card_test:
    name: Unit test (single card)
    runs-on:
      group: Fleet-H-single-card
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python: [3.11, 3.12, 3.13]
    env:
      PIP_CACHE_DIR: /home/.cache/pip
      CACHE_DIR: /home/.cache
      TASK: paddlefleet-CE-CUDA129-PYTHON-DEV-single-card-test-${{ matrix.python }}
    steps:
      - name: Determine the runner
        run: |
          gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') + 3 ))
          echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV

      - name: Check docker image and run container
        env:
          GPU_DEVICES: ${{ env.GPU_DEVICES }}
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
            -v "/dev/shm:/dev/shm"  \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}/../../..:/root \
            -v /ssd1/paddle-1/action_cache:/home/.cache \
            -v ${{ github.workspace }}:/paddle \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e repo_flag="paddlefleet" \
            -e PADDLE_ROOT \
            -e ci_scripts \
            -e CACHE_DIR \
            -e no_proxy \
            -e paddle_url \
            -e use_release \
            -e CI_name \
            -e PIP_CACHE_DIR \
            -e work_dir \
            -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
            -e GITHUB_REPO_NAME="${{ github.repository }}" \
            -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            -e GITHUB_RUN_ID="${{ github.run_id }}" \
            -w /paddle --network host ${docker_image}


      - name: create python env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          # wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
          # bash miniconda.sh -b -p $HOME/miniconda
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda init bash
          conda create -n python${{ matrix.python }} python=${{ matrix.python }} -y
          conda activate python${{ matrix.python }}
          '

      - name: Single card test
        id: basic_test
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -xce '
          pwd
          rm -rf * .[^.]*
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          export WITH_COVERAGE=ON
          pip install --upgrade pip
          git clone https://github.com/PaddlePaddle/PaddleFleet.git .
          git config --global --add safe.directory /paddle
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          python -m pip install --pre  paddlefleet --index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple
          if [ "${use_release}" == "true" ]; then
            py="${{ matrix.python }}"
            version="${py//./}"
            echo "version=$version"
            echo "Using pre-built paddle package from develop branch."
            paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
            pip install "$paddle_url$paddle_url_info" -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --force-reinstall --no-cache-dir
          fi
          pip install colorlog>=6.10.1
          pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt matplotlib==3.10.8 pytest parameterized
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          bash ci/single_card_test.sh
          single_card_exit_code=$?
          if [[ "$single_card_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mSingle card test failed.\033[0m"
            exit 1
          else
            echo -e "\033[32mSingle card test succeeded.\033[0m"
          fi
          '

      - name: Single card sonic moe test
        if: matrix.python == '3.12' && always()
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -xce '
          pwd
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          bash ci/single_card_sonic.sh
          single_card_exit_code=$?
          if [[ "$single_card_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mSingle card test failed.\033[0m"
            exit 1
          else
            echo -e "\033[32mSingle card test succeeded.\033[0m"
          fi
          '

      - name: Terminate and delete the container
        if: ${{ always() }}
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
          docker rm -f ${{ env.container_name }}

  multi-card_test:
    name: Unit test (multi-card)
    runs-on:
      group: Fleet-H-multi-card
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python: [3.11, 3.12, 3.13]
    env:
      PIP_CACHE_DIR: /home/.cache/pip
      TASK: paddlefleet-CE-CUDA129-PYTHON-DEV-multi-card_test-${{ matrix.python }}
    steps:
      - name: Check docker image and run container
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          docker run -d -t --gpus all --name ${container_name} \
            -v "/dev/shm:/dev/shm"  \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}/../../..:/root \
            -v /ssd1/paddle-1/action_cache:/home/.cache \
            -v ${{ github.workspace }}:/paddle \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e PADDLE_ROOT \
            -e ci_scripts \
            -e CACHE_DIR \
            -e no_proxy \
            -e use_release \
            -e paddle_url \
            -e CI_name \
            -e repo_flag="paddlefleet" \
            -e PIP_CACHE_DIR \
            -e work_dir \
            -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
            -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
            -e GITHUB_REPO_NAME="${{ github.repository }}" \
            -e GITHUB_EVENT_NAME="${{ github.event_name }}" \
            -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            -e GITHUB_RUN_ID="${{ github.run_id }}" \
            -w /paddle --network host ${docker_image}

      - name: create python env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          # wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
          # bash miniconda.sh -b -p $HOME/miniconda
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda init bash
          conda create -n python${{ matrix.python }} python=${{ matrix.python }} -y
          conda activate python${{ matrix.python }}
          '

      - name: Install PaddleFleet
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          rm -rf * .[^.]*
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          pip install --upgrade pip
          git clone https://github.com/PaddlePaddle/PaddleFleet.git .
          git config --global --add safe.directory /paddle
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          git submodule update --init --recursive
          pip install colorlog>=6.10.1
          python -m pip install --pre  paddlefleet --index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple
          if [ "${use_release}" == "true" ]; then
            echo "Using pre-built paddle package from develop branch."
            py="${{ matrix.python }}"
            version="${py//./}"
            echo "version=$version"
            paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
            pip install "$paddle_url$paddle_url_info" -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --force-reinstall --no-cache-dir
          fi
          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
          pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt matplotlib==3.10.8 pytest
          chmod +x /usr/local/bin/yq

          '

      - name: Multi-card test
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          export PYTHONPATH=/paddle:$PYTHONPATH
          bash ci/multi-card_test.sh
          multi_card_exit_code=$?
          if [[ "$multi_card_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mMulti card test failed.\033[0m"
            exit 1
          else
            echo -e "\033[32mMulti card test succeeded.\033[0m"
          fi
          '

      - name: Terminate and delete the container
        if: ${{ always() }}
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
          docker rm -f ${{ env.container_name }}


  integration-test-H20-single-card:
    name: Integration test (H20, single card)
    runs-on:
      group: Fleet-H-single-card
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python: [3.11, 3.12, 3.13]
    env:
      PIP_CACHE_DIR: /home/.cache/pip
      CACHE_DIR: /home/.cache
      TASK: paddlefleet-CE-CUDA129-PYTHON-DEV-integration-test-single-card-${{ matrix.python }}
    steps:
      - name: Determine the runner
        run: |
          gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') + 3 ))
          echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV

      - name: Check docker image and run container
        env:
          GPU_DEVICES: ${{ env.GPU_DEVICES }}
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          set -x
          docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
            -v "/dev/shm:/dev/shm"  \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}/../../..:/root \
            -v /ssd1/paddle-1/action_cache:/home/.cache \
            -v ${{ github.workspace }}:/workspace \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e PADDLE_ROOT \
            -e ci_scripts \
            -e CACHE_DIR \
            -e no_proxy \
            -e CI_name \
            -e paddle_url \
            -e use_release \
            -e PIP_CACHE_DIR \
            -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
            -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
            -e GITHUB_REPO_NAME="${{ github.repository }}" \
            -e GITHUB_EVENT_NAME="${{ github.event_name }}" \
            -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            -e GITHUB_RUN_ID="${{ github.run_id }}" \
            -w /workspace --network host ${docker_image}

      - name: create python env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          # wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
          # bash miniconda.sh -b -p $HOME/miniconda
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda init bash
          conda create -n python${{ matrix.python }} python=${{ matrix.python }} -y
          conda activate python${{ matrix.python }}
          '

      - name: Install PaddleFleet
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          nvidia-smi
          rm -rf * .[^.]*
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          pip install --upgrade pip
          git clone https://github.com/PaddlePaddle/PaddleFleet.git
          cd PaddleFleet
          git config --global --add safe.directory /workspace/PaddleFleet
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          pip install colorlog>=6.10.1
          python -m pip install --pre  paddlefleet --index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple
          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
          chmod +x /usr/local/bin/yq
          '

      - name: Install PaddleFormers
        id: formers_install
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          git clone -b develop https://github.com/PaddlePaddle/PaddleFormers.git
          cd PaddleFormers
          git config --global --add safe.directory /workspace/PaddleFormers
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          git log -1
          sed -i "s/from gpt_provider import GPTModelProvider/from paddleformers.transformers.gpt_provider import GPTModelProvider/g" examples/experiments/paddlefleet/glm45_provider.py
          sed -i "s/from gpt_provider import GPTModelProvider/from paddleformers.transformers.gpt_provider import GPTModelProvider/g" examples/experiments/paddlefleet/qwen_provider.py
          py="${{ matrix.python }}"
          version="${py//./}"
          echo "version=$version"
          paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
          pip install -e . --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
          if [ "${use_release}" == "true" ]; then
            py="${{ matrix.python }}"
            version="${py//./}"
            echo "version=$version"
            paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
            echo "Using pre-built paddle package from develop branch."
            pip install "$paddle_url$paddle_url_info" -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --force-reinstall --no-cache-dir
          fi
          pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
          pip install librosa==0.11.0
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          echo "paddleformers Commit"
          python -c "import paddleformers; print(paddleformers.version.commit)"
          '

      - name: Proprocess for integration test
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/preprocess.sh
          preprocess_exit_code=$?
          if [[ "$preprocess_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mPreprocess failed.\033[0m"
            exit 1
          else
            echo -e "\033[32mPreprocess succeeded.\033[0m"
          fi
          '

      - name: Integration test (GLM4.5 single-card)
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_pt_single_card.sh
          glm45_single_card_exit_code=$?
          if [[ "$glm45_single_card_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5 single-card.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5 single-card.\033[0m"
          fi
          '

      - name: Integration test (Qwen3-30B-A3B single-card)
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/qwen3_single_card.sh
          qwen3_single_card_exit_code=$?
          if [[ "$qwen3_single_card_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: Qwen3-30B-A3B single-card.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: Qwen3-30B-A3B single-card.\033[0m"
          fi
          '

      - name: Qwen3-vl-8k-single-card
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft_single_card.sh single
          '

      - name: Terminate and delete the container
        if: ${{ always() }}
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'bash PaddleFleet/ci/clean_uv_cache.sh; rm -rf * .[^.]*'
          docker rm -f ${{ env.container_name }}

  integration-test-H20-multi-card:
    name: Integration test (H20, multi-card)
    runs-on:
      group: Fleet-H-multi-card
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python: [3.11, 3.12, 3.13]
    env:
      PIP_CACHE_DIR: /home/.cache/pip
      CACHE_DIR: /home/.cache
      TASK: paddlefleet-CE-CUDA129-PYTHON-DEV-integration-test-multi-card-${{ matrix.python }}
    steps:
      - name: Check docker image and run container
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          docker run -d -t --name ${container_name} --gpus all --shm-size=32G \
            -v "/dev/shm:/dev/shm"  \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}/../../..:/root \
            -v /ssd1/paddle-1/action_cache:/home/.cache \
            -v ${{ github.workspace }}:/workspace \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e PADDLE_ROOT \
            -e ci_scripts \
            -e use_release \
            -e CACHE_DIR \
            -e no_proxy \
            -e CI_name \
            -e paddle_url \
            -e PIP_CACHE_DIR \
            -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
            -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
            -e GITHUB_REPO_NAME="${{ github.repository }}" \
            -e GITHUB_EVENT_NAME="${{ github.event_name }}" \
            -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            -e GITHUB_RUN_ID="${{ github.run_id }}" \
            -w /workspace --network host ${docker_image}

      - name: create python env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          wget https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
          bash miniconda.sh -b -p $HOME/miniconda -u
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda init bash
          conda create -n python${{ matrix.python }} python=${{ matrix.python }} -y
          conda activate python${{ matrix.python }}
          '

      - name: Install PaddleFleet
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          rm -rf * .[^.]*
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda init bash
          conda activate python${{ matrix.python }}
          pip install --upgrade pip
          git clone https://github.com/PaddlePaddle/PaddleFleet.git
          cd PaddleFleet
          git config --global --add safe.directory /workspace/PaddleFleet
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          pip install colorlog>=6.10.1
          python -m pip install --pre  paddlefleet --index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple
          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
          chmod +x /usr/local/bin/yq
          '

      - name: Install PaddleFormers
        id: formers_install
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          git clone -b develop https://github.com/PaddlePaddle/PaddleFormers.git
          cd PaddleFormers
          cp examples/experiments/paddlefleet/glm45.json examples/experiments/paddlefleet/glm45_fp8.json
          git config --global --add safe.directory /workspace/PaddleFormers
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          git log -1
          pip install -e . --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
          if [ "${use_release}" == "true" ]; then
            py="${{ matrix.python }}"
            version="${py//./}"
            echo "version=$version"
            paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
            echo "Using pre-built paddle package from develop branch."
            pip install "$paddle_url$paddle_url_info" -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --force-reinstall --no-cache-dir
          fi
          pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
          pip install bce-python-sdk==0.8.74
          pip install coverage==7.6.1
          pip install librosa==0.11.0
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          echo "paddleformers Commit"
          python -c "import paddleformers; print(paddleformers.version.commit)"
          '

      - name: GLM4.5 pre-train
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_pt.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5.\033[0m"
          fi
          '

      - name: GLM4.5 sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_sft.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5.\033[0m"
          fi
          '

      - name: GLM4.5 lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_lora.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5.\033[0m"
          fi
          '

      - name: GLM4.5 dpo
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_dpo.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5 dpo.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5 dpo.\033[0m"
          fi
          '

      - name: GLM4.5 dpo_lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_dpo_lora.sh
          '

      - name: Integration test (GLM4.5 multi-card FP8)
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_pt_fp8.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5 FP8.\033[0m"
          fi
          '


      - name: GLM4.5 pre-train (Grouped GEMM)
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_pt_grouped_gemm.sh
          glm45_exit_code=$?
          if [[ "$glm45_exit_code" != "0" ]]; then
            echo -e "::error:: \033[31mIntegration test failed: GLM4.5 Grouped GEMM.\033[0m"
            exit 1
          else
            echo -e "\033[32mIntegration test succeeded: GLM4.5 Grouped GEMM.\033[0m"
          fi
          '

      - name: GLM4.5 pre-train (EP4)
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_pt_ep4.sh
          '

      - name: Qwen pre-train
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/qwen.sh pt
          '

      - name: Qwen sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/qwen.sh sft
          '

      - name: Qwen lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/qwen.sh lora
          '
      - name: Qwen vl sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft.sh tp8 h20
          '

      - name: Qwen vl lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3vl_lora.sh h20
          '

      - name: Qwen vl moe
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 10m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft.sh moe h20
          '

      - name: Qwen3-vl-8k-fsdp
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 10m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft.sh fsdp h20
          '

      - name: Terminate and delete the container
        if: ${{ always() }}
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'bash PaddleFleet/ci/clean_uv_cache.sh; rm -rf * .[^.]*'
          docker rm -f ${{ env.container_name }}


  integration-test-a100:
    name: Integration test (A100)
    runs-on:
      group: Distribute
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python: [3.11, 3.12, 3.13]
    env:
      PIP_CACHE_DIR: /home/.cache/pip
      CACHE_DIR: /home/.cache
      TASK: paddlefleet-CE-CUDA129-PYTHON-DEV-integration-test-A100-${{ matrix.python }}
    steps:
      - name: Check docker image and run container
        run: |
          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
          echo "container_name=${container_name}" >> ${{ github.env }}
          docker pull $docker_image
          docker run -d -t --name ${container_name} --gpus all --shm-size=32G \
            -v "/dev/shm:/dev/shm"  \
            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
            -v ${{ github.workspace }}/../../..:/root \
            -v /ssd1/paddle-1/action_cache:/home/.cache \
            -v ${{ github.workspace }}:/workspace \
            -e BRANCH \
            -e PR_ID \
            -e COMMIT_ID \
            -e PADDLE_ROOT \
            -e ci_scripts \
            -e CACHE_DIR \
            -e paddle_url \
            -e use_release \
            -e no_proxy \
            -e CI_name \
            -e PIP_CACHE_DIR \
            -e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
            -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
            -e GITHUB_REPO_NAME="${{ github.repository }}" \
            -e GITHUB_EVENT_NAME="${{ github.event_name }}" \
            -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            -e GITHUB_RUN_ID="${{ github.run_id }}" \
            -w /workspace --network host ${docker_image}

      - name: create python env
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
          conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
          conda init bash
          conda create -n python${{ matrix.python }} python=${{ matrix.python }} -y
          conda activate python${{ matrix.python }}
          '

      - name: Install PaddleFleet
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          rm -rf * .[^.]*
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          pip install --upgrade pip
          git clone https://github.com/PaddlePaddle/PaddleFleet.git
          cd PaddleFleet
          git config --global --add safe.directory /workspace/PaddleFleet
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          pip install colorlog>=6.10.1
          pip install colorlog>=6.10.1
          python -m pip install --pre  paddlefleet --index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url https://pypi.tuna.tsinghua.edu.cn/simple
          wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
          chmod +x /usr/local/bin/yq
          '

      - name: Install PaddleFormers
        id: formers_install
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          git clone -b develop https://github.com/PaddlePaddle/PaddleFormers.git
          cd PaddleFormers
          git config --global --add safe.directory /workspace/PaddleFormers
          git config user.name "PaddleCI"
          git config user.email "paddle_ci@example.com"
          git config pull.rebase false
          git log -1
          pip install -e . --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
          if [ "${use_release}" == "true" ]; then
            py="${{ matrix.python }}"
            version="${py//./}"
            echo "version=$version"
            paddle_url_info=paddlepaddle_gpu-0.0.0-cp${version}-cp${version}-linux_x86_64.whl
            echo "Using pre-built paddle package from develop branch."
            pip install "$paddle_url$paddle_url_info" -i https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --force-reinstall --no-cache-dir
          fi
          pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
          pip install bce-python-sdk==0.8.74
          pip install coverage==7.6.1
          pip install librosa==0.11.0
          echo "Paddle Commit"
          python -c "import paddle; print(paddle.version.commit)"
          echo "PaddleFleet Commit"
          python -c "import paddlefleet; print(paddlefleet.version.commit)"
          echo "paddleformers Commit"
          python -c "import paddleformers; print(paddleformers.version.commit)"
          '

      - name: GLM4.5 pre-train
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          bash -x PaddleFormers/tests/integration_test/glm45_a100.sh pt
          '

      - name: GLM4.5 sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_a100.sh sft
          '

      - name: GLM4.5 lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_a100.sh lora
          '

      - name: GLM4.5 dpo
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_a100.sh dpo
          glm45_exit_code=$?
          '

      - name: GLM4.5 dpo_lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/glm45_a100.sh dpo_lora
          '

      - name: Qwen pre-train
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3_a100.sh pt
          '

      - name: Qwen sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3_a100.sh sft
          '

      - name: Qwen lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3_a100.sh lora
          '


      - name: Qwen vl sft
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft.sh tp8 a100
          '

      - name: Qwen vl lora
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 5m bash -x PaddleFormers/tests/integration_test/qwen3vl_lora.sh a100
          '


      - name: Qwen vl moe
        if: (success() || failure()) && steps.formers_install.conclusion == 'success'
        run: |
          docker exec -t ${{ env.container_name }} /bin/bash -ce '
          source /root/proxy
          source "$HOME/miniconda/etc/profile.d/conda.sh"
          conda activate python${{ matrix.python }}
          timeout 10m bash -x PaddleFormers/tests/integration_test/qwen3vl_sft.sh moe a100
          '


      - name: Terminate and delete the container
        if: ${{ always() }}
        run: |
          set +e
          docker exec -t ${{ env.container_name }} /bin/bash -c 'bash PaddleFleet/ci/clean_uv_cache.sh; rm -rf * .[^.]*'
          docker rm -f ${{ env.container_name }}