diff --git a/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml b/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml index 149a0f211..171eacf62 100644 --- a/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml +++ b/pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml @@ -72,6 +72,16 @@ spec: limits: memory: 128Mi cpu: 100m + - name: utils + image: ghcr.io/pingcap-qe/cd/utils/release:v2025.10.12-7-gfdd779c + tty: true + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + cpu: "1" + memory: 4Gi volumes: - emptyDir: {} name: volume-0 diff --git a/pipelines/tikv/migration/latest/pod-pull_integration_test.yaml b/pipelines/tikv/migration/latest/pod-pull_integration_test.yaml index 6c4a2696a..b9ec82e0a 100644 --- a/pipelines/tikv/migration/latest/pod-pull_integration_test.yaml +++ b/pipelines/tikv/migration/latest/pod-pull_integration_test.yaml @@ -28,6 +28,16 @@ spec: limits: memory: 256Mi cpu: 100m + - name: utils + image: ghcr.io/pingcap-qe/cd/utils/release:v2025.10.12-7-gfdd779c + tty: true + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + cpu: "1" + memory: 4Gi affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy b/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy index f78271873..dda18d391 100644 --- a/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy +++ b/pipelines/tikv/migration/latest/pull_integration_kafka_test.groovy @@ -8,6 +8,11 @@ final GIT_FULL_REPO_NAME = 'tikv/migration' final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' final POD_TEMPLATE_FILE = 'pipelines/tikv/migration/latest/pod-pull_integration_kafka_test.yaml' final REFS = readJSON(text: params.JOB_SPEC).refs +final OCI_TAG_TIDB = component.computeArtifactOciTagFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_ETCD = 'v3.5.15' +final OCI_TAG_YCSB = 'v1.0.3' pipeline { agent { @@ -18,7 +23,11 @@ pipeline { } } environment { - FILE_SERVER_URL = 'http://fileserver.pingcap.net' + // OCI artifact registry: hub-zot.pingcap.net/mirrors/hub + // tidb-server: hub-zot.pingcap.net/mirrors/hub/pingcap/tidb/package:_linux_amd64 + // tikv-server: hub-zot.pingcap.net/mirrors/hub/tikv/tikv/package:_linux_amd64 + // pd-server: hub-zot.pingcap.net/mirrors/hub/tikv/pd/package:_linux_amd64 + OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub' } options { timeout(time: 65, unit: 'MINUTES') @@ -66,10 +75,26 @@ pipeline { steps { dir('migration') { cache(path: "./cdc", includes: '**/*', key: "ws/${BUILD_TAG}/tikvcdc") { + container("utils") { + sh label: 'download test binaries via OCI', script: """ + mkdir -p ./cdc/scripts/bin + cd ./cdc/scripts/bin + ${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \ + --tidb=${OCI_TAG_TIDB} \ + --tikv=${OCI_TAG_TIKV} \ + --pd=${OCI_TAG_PD} \ + --pd-ctl=${OCI_TAG_PD} \ + --etcdctl=${OCI_TAG_ETCD} \ + --ycsb=${OCI_TAG_YCSB} + chmod +x tidb-server tikv-server pd-server pd-ctl etcdctl go-ycsb + cd ../../ + touch prepare_test_binaries + ls -alh + """ + } container("golang") { sh label: 'integration test prepare', script: """#!/usr/bin/env bash cd cdc/ - make prepare_test_binaries make check_third_party_binary make integration_test_build """ diff --git a/pipelines/tikv/migration/latest/pull_integration_test.groovy b/pipelines/tikv/migration/latest/pull_integration_test.groovy index a74a85aa2..7c8e7b334 100644 --- a/pipelines/tikv/migration/latest/pull_integration_test.groovy +++ b/pipelines/tikv/migration/latest/pull_integration_test.groovy @@ -8,6 +8,11 @@ final GIT_FULL_REPO_NAME = 'tikv/migration' final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' final POD_TEMPLATE_FILE = 'pipelines/tikv/migration/latest/pod-pull_integration_test.yaml' final REFS = readJSON(text: params.JOB_SPEC).refs +final OCI_TAG_TIDB = component.computeArtifactOciTagFromPR('tidb', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, 'master') +final OCI_TAG_ETCD = 'v3.5.15' +final OCI_TAG_YCSB = 'v1.0.3' pipeline { agent { @@ -18,7 +23,11 @@ pipeline { } } environment { - FILE_SERVER_URL = 'http://fileserver.pingcap.net' + // OCI artifact registry: hub-zot.pingcap.net/mirrors/hub + // tidb-server: hub-zot.pingcap.net/mirrors/hub/pingcap/tidb/package:_linux_amd64 + // tikv-server: hub-zot.pingcap.net/mirrors/hub/tikv/tikv/package:_linux_amd64 + // pd-server: hub-zot.pingcap.net/mirrors/hub/tikv/pd/package:_linux_amd64 + OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub' } options { timeout(time: 65, unit: 'MINUTES') @@ -66,10 +75,26 @@ pipeline { steps { dir('migration') { cache(path: "./cdc", includes: '**/*', key: "ws/${BUILD_TAG}/tikvcdc") { + container("utils") { + sh label: 'download test binaries via OCI', script: """ + mkdir -p ./cdc/scripts/bin + cd ./cdc/scripts/bin + ${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \ + --tidb=${OCI_TAG_TIDB} \ + --tikv=${OCI_TAG_TIKV} \ + --pd=${OCI_TAG_PD} \ + --pd-ctl=${OCI_TAG_PD} \ + --etcdctl=${OCI_TAG_ETCD} \ + --ycsb=${OCI_TAG_YCSB} + chmod +x tidb-server tikv-server pd-server pd-ctl etcdctl go-ycsb + cd ../../ + touch prepare_test_binaries + ls -alh + """ + } container("golang") { sh label: 'integration test prepare', script: """#!/usr/bin/env bash cd cdc/ - make prepare_test_binaries make check_third_party_binary make integration_test_build """ diff --git a/prow-jobs/tikv/migration/latest-presubmits.yaml b/prow-jobs/tikv/migration/latest-presubmits.yaml index 68c3026de..c2dfb645c 100644 --- a/prow-jobs/tikv/migration/latest-presubmits.yaml +++ b/prow-jobs/tikv/migration/latest-presubmits.yaml @@ -1,29 +1,27 @@ +global_definitions: + brancher: &brancher + branches: + - ^main$ + - ^cdc-release-.*$ + - ^br-release-.*$ + jenkins_job: &jenkins_job + agent: jenkins + labels: + master: "0" + decorate: false + # struct ref: https://pkg.go.dev/sigs.k8s.io/prow/pkg/config#Presubmit presubmits: tikv/migration: - - name: tikv/migration/pull_integration_test - agent: jenkins - labels: - master: "0" - decorate: false # need add this. + - <<: [*brancher, *jenkins_job] + name: tikv/migration/pull_integration_test always_run: true context: pull-integration-test trigger: "(?m)^/test (?:.*? )?pull-integration-test(?: .*?)?$" rerun_command: "/test pull-integration-test" - branches: - - ^main$ - - ^cdc-release-.*$ - - ^br-release-.*$ - - name: tikv/migration/pull_integration_kafka_test - agent: jenkins - labels: - master: "0" - decorate: false # need add this. + - <<: [*brancher, *jenkins_job] + name: tikv/migration/pull_integration_kafka_test always_run: true context: pull-integration-kafka-test trigger: "(?m)^/test (?:.*? )?pull-integration-kafka-test(?: .*?)?$" rerun_command: "/test pull-integration-kafka-test" - branches: - - ^main$ - - ^cdc-release-.*$ - - ^br-release-.*$