feat: 新增devcloud构建机接入 #8265

pom.xml

@@ -73,6 +73,11 @@
             <artifactId>commons-lang3</artifactId>
             <version>${lib.commons.lang3.version}</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.15</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

src/main/kotlin/com/tencent/bk/devops/plugin/docker/DevCloudExecutor.kt

@@ -0,0 +1,212 @@
+package com.tencent.bk.devops.plugin.docker
+
+import com.tencent.bk.devops.plugin.docker.pojo.DockerRunLogRequest
+import com.tencent.bk.devops.plugin.docker.pojo.DockerRunLogResponse
+import com.tencent.bk.devops.plugin.docker.pojo.DockerRunRequest
+import com.tencent.bk.devops.plugin.docker.pojo.DockerRunResponse
+import com.tencent.bk.devops.plugin.docker.pojo.common.DockerStatus
+import com.tencent.bk.devops.plugin.docker.pojo.job.request.JobParam
+import com.tencent.bk.devops.plugin.docker.pojo.job.request.JobRequest
+import com.tencent.bk.devops.plugin.docker.pojo.job.request.Registry
+import com.tencent.bk.devops.plugin.docker.pojo.status.JobStatusResponse
+import com.tencent.bk.devops.plugin.docker.utils.DevCloudClient
+import com.tencent.bk.devops.plugin.docker.utils.EnvUtils
+import com.tencent.bk.devops.plugin.docker.utils.ParamUtils.beiJ2UTC
+import org.apache.commons.lang3.RandomUtils
+import org.apache.tools.ant.types.Commandline
+import org.slf4j.LoggerFactory
+
+object DevCloudExecutor {
+    private val VOLUME_SERVER = "volume_server"
+    private val VOLUME_PATH = "volume_path"
+    private val VOLUME_MOUNT_PATH = "volume_mount_path"
+
+    private val logger = LoggerFactory.getLogger(DevCloudExecutor::class.java)
+
+    fun execute(request: DockerRunRequest): DockerRunResponse {
+        val startTimeStamp = System.currentTimeMillis()
+        val jobRequest = getJobRequest(request)
+        val devCloudClient = DevCloudClient(
+            executeUser = request.userId,
+            devCloudAppId = request.extraOptions?.get("devCloudAppId") ?: throw RuntimeException("devCloudAppId is null"),
+            devCloudUrl = request.extraOptions["devCloudUrl"] ?: throw RuntimeException("devCloudUrl is null"),
+            devCloudToken = request.extraOptions["devCloudToken"] ?: throw RuntimeException("devCloudToken is null")
+        )
+        val task = devCloudClient.createJob(jobRequest)
+
+        return DockerRunResponse(
+            extraOptions = request.extraOptions.plus(mapOf(
+                "devCloudTaskId" to task.taskId.toString(),
+                "devCloudJobName" to task.jobName,
+                "startTimeStamp" to startTimeStamp.toString()
+            ))
+        )
+    }
+
+    fun getLogs(param: DockerRunLogRequest): DockerRunLogResponse {
+        val extraOptions = param.extraOptions.toMutableMap()
+
+        val devCloudClient = DevCloudClient(
+            executeUser = param.userId,
+            devCloudAppId = param.extraOptions["devCloudAppId"] ?: throw RuntimeException("devCloudAppId is null"),
+            devCloudUrl = param.extraOptions["devCloudUrl"] ?: throw RuntimeException("devCloudUrl is null"),
+            devCloudToken = param.extraOptions["devCloudToken"] ?: throw RuntimeException("devCloudToken is null")
+        )
+
+        // get task status
+        val taskId = param.extraOptions["devCloudTaskId"] ?: throw RuntimeException("devCloudTaskId is null")
+        val taskStatusFlag = param.extraOptions["taskStatusFlag"]
+        if (taskStatusFlag.isNullOrBlank() || taskStatusFlag == DockerStatus.running) {
+            val taskStatus = devCloudClient.getTaskStatus(taskId.toInt())
+            if (taskStatus.status == "failed") {
+                return DockerRunLogResponse(
+                    status = DockerStatus.failure,
+                    message = "get task status fail",
+                    extraOptions = extraOptions
+                )
+            }
+            if (taskStatus.status != "succeeded") {
+                return DockerRunLogResponse(
+                    status = DockerStatus.running,
+                    message = "get task status...",
+                    extraOptions = extraOptions
+                )
+            }
+        }
+        extraOptions["taskStatusFlag"] = DockerStatus.success
+
+        // get job status
+        val jobStatusFlag = param.extraOptions["jobStatusFlag"]
+        val jobName = param.extraOptions["devCloudJobName"] ?: throw RuntimeException("devCloudJobName is null")
+        var jobStatusResp: JobStatusResponse? = null
+        var jobIp = ""
+        if (jobStatusFlag.isNullOrBlank() || jobStatusFlag == DockerStatus.running) {
+            jobStatusResp = devCloudClient.getJobStatus(jobName)
+            jobIp = jobStatusResp.data.pod_result!![0].ip ?: ""
+            val jobStatus = jobStatusResp.data.status
+            if ("failed" != jobStatus && "succeeded" != jobStatus && "running" != jobStatus) {
+                return DockerRunLogResponse(
+                    status = DockerStatus.running,
+                    message = "get job status...",
+                    extraOptions = extraOptions
+                )
+            }
+        }
+        extraOptions["jobIp"] = jobIp
+        extraOptions["jobStatusFlag"] = DockerStatus.success
+
+        // actual get log logic
+        val startTimeStamp = extraOptions["startTimeStamp"]?.toLong() ?: System.currentTimeMillis()
+        val logs = mutableListOf<String>()
+
+        val logResult = devCloudClient.getLog(jobName, beiJ2UTC(startTimeStamp))
+
+        // only if not blank then add start time
+        val isNotBlank = logResult.isNullOrBlank()
+        if (!isNotBlank) extraOptions["startTimeStamp"] = (startTimeStamp + param.timeGap).toString()
+
+        // add logs
+        if (!isNotBlank) logs.add(logResult!!)
+
+
+        if (jobStatusResp == null) {
+            jobStatusResp = devCloudClient.getJobStatus(jobName)
+        }
+        val finalStatus = jobStatusResp
+        val podResults = finalStatus.data.pod_result
+        podResults?.forEach { ps ->
+            ps.events?.forEach { event ->
+                // add logs
+                logs.add(event.message)
+            }
+        }
+
+        if (finalStatus.data.status in listOf("failed", "succeeded")) {
+            val url = "/api/v2.1/job/$jobName/status"
+            logger.info("final job status url: $url")
+            logger.info("final job status data: $jobStatusResp")
+            Thread.sleep(6000)
+            val finalLogs = devCloudClient.getLog(jobName, beiJ2UTC(startTimeStamp + 6000))
+            if (finalStatus.data.status == "failed") {
+                return DockerRunLogResponse(
+                    log = logs.plus(finalLogs ?: ""),
+                    status = DockerStatus.failure,
+                    message = "docker run fail...",
+                    extraOptions = extraOptions
+                )
+            }
+            return DockerRunLogResponse(
+                log = logs.plus(finalLogs ?: ""),
+                status = DockerStatus.success,
+                message = "docker run success...",
+                extraOptions = extraOptions
+            )
+        }
+
+        return DockerRunLogResponse(
+            log = logs,
+            status = DockerStatus.running,
+            message = "get log...",
+            extraOptions = extraOptions
+        )
+    }
+
+    private fun getJobRequest(param: DockerRunRequest): JobRequest {
+        with(param) {
+            // get job param
+            val cmdTmp = mutableListOf<String>()
+            command.forEach {
+                cmdTmp.add(it.removePrefix("\"").removeSuffix("\"").removePrefix("\'").removeSuffix("\'"))
+            }
+            val cmd = if (cmdTmp.size == 1) { Commandline.translateCommandline(cmdTmp.first()).toList() } else { cmdTmp }
+            val jobParam = JobParam(
+                env = envMap,
+                command = cmd,
+                labels = labels,
+                ipEnabled = ipEnabled
+            )
+
+            if (jobParam.nfsVolume == null) {
+                val volumeServer = System.getenv(VOLUME_SERVER)
+                if (!volumeServer.isNullOrBlank()) {
+                    jobParam.nfsVolume = listOf(
+                        JobParam.NfsVolume(
+                            System.getenv(VOLUME_SERVER),
+                            System.getenv(VOLUME_PATH),
+                            System.getenv(VOLUME_MOUNT_PATH)
+                        )
+                    )
+                }
+            }
+
+            // get docker image host & path
+            val imagePair = getImagePair(param.imageName)
+
+            // get user pass param
+            val registry = Registry(
+                host = imagePair.first,
+                username = param.dockerLoginUsername,
+                password = param.dockerLoginPassword
+            )
+
+            return JobRequest(
+                alias = "bkdevops_job_${System.currentTimeMillis()}_${RandomUtils.nextLong()}",
+                regionId = "",
+                clusterType = "normal",
+                activeDeadlineSeconds = 86400,
+                image = imagePair.second,
+                registry = registry,
+                cpu = 1,
+                memory = "1024M",
+                params = jobParam,
+                podNameSelector = EnvUtils.getHostName()
+            )
+        }
+    }
+
+    private fun getImagePair(imageName: String): Pair<String, String> {
+        val targetImageRepo = imageName.split("/").first()
+        val targetImageName = imageName.removePrefix(targetImageRepo).removeSuffix("/")
+        return Pair(targetImageRepo, targetImageName)
+    }
+}

src/main/kotlin/com/tencent/bk/devops/plugin/docker/DockerApi.kt

@@ -37,6 +37,7 @@ open class DockerApi : BaseApi() {
                 response = when {
                     "docker" == property -> CommonExecutor.execute(projectId, pipelineId, buildId, param, taskId)
                     "KUBERNETES" == jobPoolType -> KubernetesExecutor.execute(param)
+                    "PUBLIC_DEVCLOUD" == jobPoolType -> DevCloudExecutor.execute(param)
                     else -> ThirdPartExecutor.execute(param)
                 }
             }
@@ -72,6 +73,7 @@ open class DockerApi : BaseApi() {
                 response = when {
                     "docker" == property -> CommonExecutor.getLogs(projectId, pipelineId, buildId, param)
                     "KUBERNETES" == jobPoolType -> KubernetesExecutor.getLogs(param)
+                    "PUBLIC_DEVCLOUD" == jobPoolType -> DevCloudExecutor.getLogs(param)
                     else -> ThirdPartExecutor.getLogs(param)
                 }
             }

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/DevCloudTask.kt

@@ -0,0 +1,6 @@
+package com.tencent.bk.devops.plugin.docker.pojo
+
+data class DevCloudTask(
+    val taskId: Int,
+    val jobName: String
+)

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobParam.kt

@@ -0,0 +1,20 @@
+package com.tencent.bk.devops.plugin.docker.pojo.job.request
+
+import com.fasterxml.jackson.annotation.JsonInclude
+
+data class JobParam(
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    var env: Map<String, String>? = null,
+    val command: List<String>? = null,
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    var nfsVolume: List<NfsVolume>? = null,
+    var workDir: String? = "/data/landun/workspace",
+    var labels: Map<String, String>? = emptyMap(),
+    var ipEnabled: Boolean? = true
+) {
+    data class NfsVolume(
+        val server: String? = null,
+        val path: String? = null,
+        val mountPath: String? = null
+    )
+}

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobRequest.kt

@@ -0,0 +1,14 @@
+package com.tencent.bk.devops.plugin.docker.pojo.job.request
+
+data class JobRequest(
+    val alias: String? = null,
+    val regionId: String,
+    val clusterType: String? = null,
+    val activeDeadlineSeconds: Int? = null,
+    val image: String? = null,
+    val registry: Registry? = null,
+    val cpu: Int? = null,
+    val memory: String? = null,
+    val params: JobParam? = null,
+    val podNameSelector: String? = null
+)

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/PcgJobRequest.kt

@@ -0,0 +1,16 @@
+package com.tencent.bk.devops.plugin.docker.pojo.job.request
+
+
+data class PcgJobRequest(
+    val alias: String? = null,
+    val regionId: String,
+    val clusterType: String? = null,
+    val activeDeadlineSeconds: Int? = null,
+    val image: String? = null,
+    val registry: Registry? = null,
+    val cpu: Int? = null,
+    val memory: String? = null,
+    val params: JobParam? = null,
+    val podNameSelector: String? = null,
+    val operator: String? = null
+)

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/Registry.kt

@@ -0,0 +1,7 @@
+package com.tencent.bk.devops.plugin.docker.pojo.job.request
+
+data class Registry (
+    val host: String,
+    val username: String?,
+    val password: String?
+)

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/response/JobResponse.kt

@@ -0,0 +1,12 @@
+package com.tencent.bk.devops.plugin.docker.pojo.job.response
+
+data class JobResponse (
+    val actionCode: Int,
+    val actionMessage: String,
+    val data: JobResponseData
+) {
+    data class JobResponseData (
+        val name: String,
+        val taskId: Int
+    )
+}

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusData.kt

@@ -0,0 +1,22 @@
+package com.tencent.bk.devops.plugin.docker.pojo.status
+
+data class JobStatusData(
+    val deleted: Boolean,
+    val status: String,
+    val pod_result: List<PodResult>?
+) {
+    data class PodResult(
+        val ip: String?,
+        val events: List<PodResultEvent>?
+    )
+
+    data class PodResultEvent(
+        val message: String,
+        val reason: String,
+        val type: String
+    ) {
+        override fun toString(): String {
+            return "reason: $reason, type: $type"
+        }
+    }
+}

src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusResponse.kt

@@ -0,0 +1,8 @@
+package com.tencent.bk.devops.plugin.docker.pojo.status
+
+
+data class JobStatusResponse (
+    val actionCode: Int,
+    val actionMessage: String,
+    val data: JobStatusData
+)