diff --git a/pom.xml b/pom.xml index aad03b0..dc626f8 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,11 @@ commons-lang3 ${lib.commons.lang3.version} + + commons-codec + commons-codec + 1.15 + org.slf4j slf4j-api diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DevCloudExecutor.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DevCloudExecutor.kt new file mode 100644 index 0000000..8640573 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DevCloudExecutor.kt @@ -0,0 +1,212 @@ +package com.tencent.bk.devops.plugin.docker + +import com.tencent.bk.devops.plugin.docker.pojo.DockerRunLogRequest +import com.tencent.bk.devops.plugin.docker.pojo.DockerRunLogResponse +import com.tencent.bk.devops.plugin.docker.pojo.DockerRunRequest +import com.tencent.bk.devops.plugin.docker.pojo.DockerRunResponse +import com.tencent.bk.devops.plugin.docker.pojo.common.DockerStatus +import com.tencent.bk.devops.plugin.docker.pojo.job.request.JobParam +import com.tencent.bk.devops.plugin.docker.pojo.job.request.JobRequest +import com.tencent.bk.devops.plugin.docker.pojo.job.request.Registry +import com.tencent.bk.devops.plugin.docker.pojo.status.JobStatusResponse +import com.tencent.bk.devops.plugin.docker.utils.DevCloudClient +import com.tencent.bk.devops.plugin.docker.utils.EnvUtils +import com.tencent.bk.devops.plugin.docker.utils.ParamUtils.beiJ2UTC +import org.apache.commons.lang3.RandomUtils +import org.apache.tools.ant.types.Commandline +import org.slf4j.LoggerFactory + +object DevCloudExecutor { + private val VOLUME_SERVER = "volume_server" + private val VOLUME_PATH = "volume_path" + private val VOLUME_MOUNT_PATH = "volume_mount_path" + + private val logger = LoggerFactory.getLogger(DevCloudExecutor::class.java) + + fun execute(request: DockerRunRequest): DockerRunResponse { + val startTimeStamp = System.currentTimeMillis() + val jobRequest = getJobRequest(request) + val devCloudClient = DevCloudClient( + executeUser = request.userId, + devCloudAppId = request.extraOptions?.get("devCloudAppId") ?: throw RuntimeException("devCloudAppId is null"), + devCloudUrl = request.extraOptions["devCloudUrl"] ?: throw RuntimeException("devCloudUrl is null"), + devCloudToken = request.extraOptions["devCloudToken"] ?: throw RuntimeException("devCloudToken is null") + ) + val task = devCloudClient.createJob(jobRequest) + + return DockerRunResponse( + extraOptions = request.extraOptions.plus(mapOf( + "devCloudTaskId" to task.taskId.toString(), + "devCloudJobName" to task.jobName, + "startTimeStamp" to startTimeStamp.toString() + )) + ) + } + + fun getLogs(param: DockerRunLogRequest): DockerRunLogResponse { + val extraOptions = param.extraOptions.toMutableMap() + + val devCloudClient = DevCloudClient( + executeUser = param.userId, + devCloudAppId = param.extraOptions["devCloudAppId"] ?: throw RuntimeException("devCloudAppId is null"), + devCloudUrl = param.extraOptions["devCloudUrl"] ?: throw RuntimeException("devCloudUrl is null"), + devCloudToken = param.extraOptions["devCloudToken"] ?: throw RuntimeException("devCloudToken is null") + ) + + // get task status + val taskId = param.extraOptions["devCloudTaskId"] ?: throw RuntimeException("devCloudTaskId is null") + val taskStatusFlag = param.extraOptions["taskStatusFlag"] + if (taskStatusFlag.isNullOrBlank() || taskStatusFlag == DockerStatus.running) { + val taskStatus = devCloudClient.getTaskStatus(taskId.toInt()) + if (taskStatus.status == "failed") { + return DockerRunLogResponse( + status = DockerStatus.failure, + message = "get task status fail", + extraOptions = extraOptions + ) + } + if (taskStatus.status != "succeeded") { + return DockerRunLogResponse( + status = DockerStatus.running, + message = "get task status...", + extraOptions = extraOptions + ) + } + } + extraOptions["taskStatusFlag"] = DockerStatus.success + + // get job status + val jobStatusFlag = param.extraOptions["jobStatusFlag"] + val jobName = param.extraOptions["devCloudJobName"] ?: throw RuntimeException("devCloudJobName is null") + var jobStatusResp: JobStatusResponse? = null + var jobIp = "" + if (jobStatusFlag.isNullOrBlank() || jobStatusFlag == DockerStatus.running) { + jobStatusResp = devCloudClient.getJobStatus(jobName) + jobIp = jobStatusResp.data.pod_result!![0].ip ?: "" + val jobStatus = jobStatusResp.data.status + if ("failed" != jobStatus && "succeeded" != jobStatus && "running" != jobStatus) { + return DockerRunLogResponse( + status = DockerStatus.running, + message = "get job status...", + extraOptions = extraOptions + ) + } + } + extraOptions["jobIp"] = jobIp + extraOptions["jobStatusFlag"] = DockerStatus.success + + // actual get log logic + val startTimeStamp = extraOptions["startTimeStamp"]?.toLong() ?: System.currentTimeMillis() + val logs = mutableListOf() + + val logResult = devCloudClient.getLog(jobName, beiJ2UTC(startTimeStamp)) + + // only if not blank then add start time + val isNotBlank = logResult.isNullOrBlank() + if (!isNotBlank) extraOptions["startTimeStamp"] = (startTimeStamp + param.timeGap).toString() + + // add logs + if (!isNotBlank) logs.add(logResult!!) + + + if (jobStatusResp == null) { + jobStatusResp = devCloudClient.getJobStatus(jobName) + } + val finalStatus = jobStatusResp + val podResults = finalStatus.data.pod_result + podResults?.forEach { ps -> + ps.events?.forEach { event -> + // add logs + logs.add(event.message) + } + } + + if (finalStatus.data.status in listOf("failed", "succeeded")) { + val url = "/api/v2.1/job/$jobName/status" + logger.info("final job status url: $url") + logger.info("final job status data: $jobStatusResp") + Thread.sleep(6000) + val finalLogs = devCloudClient.getLog(jobName, beiJ2UTC(startTimeStamp + 6000)) + if (finalStatus.data.status == "failed") { + return DockerRunLogResponse( + log = logs.plus(finalLogs ?: ""), + status = DockerStatus.failure, + message = "docker run fail...", + extraOptions = extraOptions + ) + } + return DockerRunLogResponse( + log = logs.plus(finalLogs ?: ""), + status = DockerStatus.success, + message = "docker run success...", + extraOptions = extraOptions + ) + } + + return DockerRunLogResponse( + log = logs, + status = DockerStatus.running, + message = "get log...", + extraOptions = extraOptions + ) + } + + private fun getJobRequest(param: DockerRunRequest): JobRequest { + with(param) { + // get job param + val cmdTmp = mutableListOf() + command.forEach { + cmdTmp.add(it.removePrefix("\"").removeSuffix("\"").removePrefix("\'").removeSuffix("\'")) + } + val cmd = if (cmdTmp.size == 1) { Commandline.translateCommandline(cmdTmp.first()).toList() } else { cmdTmp } + val jobParam = JobParam( + env = envMap, + command = cmd, + labels = labels, + ipEnabled = ipEnabled + ) + + if (jobParam.nfsVolume == null) { + val volumeServer = System.getenv(VOLUME_SERVER) + if (!volumeServer.isNullOrBlank()) { + jobParam.nfsVolume = listOf( + JobParam.NfsVolume( + System.getenv(VOLUME_SERVER), + System.getenv(VOLUME_PATH), + System.getenv(VOLUME_MOUNT_PATH) + ) + ) + } + } + + // get docker image host & path + val imagePair = getImagePair(param.imageName) + + // get user pass param + val registry = Registry( + host = imagePair.first, + username = param.dockerLoginUsername, + password = param.dockerLoginPassword + ) + + return JobRequest( + alias = "bkdevops_job_${System.currentTimeMillis()}_${RandomUtils.nextLong()}", + regionId = "", + clusterType = "normal", + activeDeadlineSeconds = 86400, + image = imagePair.second, + registry = registry, + cpu = 1, + memory = "1024M", + params = jobParam, + podNameSelector = EnvUtils.getHostName() + ) + } + } + + private fun getImagePair(imageName: String): Pair { + val targetImageRepo = imageName.split("/").first() + val targetImageName = imageName.removePrefix(targetImageRepo).removeSuffix("/") + return Pair(targetImageRepo, targetImageName) + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DockerApi.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DockerApi.kt index 697ec1a..9581b95 100644 --- a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DockerApi.kt +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/DockerApi.kt @@ -37,6 +37,7 @@ open class DockerApi : BaseApi() { response = when { "docker" == property -> CommonExecutor.execute(projectId, pipelineId, buildId, param, taskId) "KUBERNETES" == jobPoolType -> KubernetesExecutor.execute(param) + "PUBLIC_DEVCLOUD" == jobPoolType -> DevCloudExecutor.execute(param) else -> ThirdPartExecutor.execute(param) } } @@ -72,6 +73,7 @@ open class DockerApi : BaseApi() { response = when { "docker" == property -> CommonExecutor.getLogs(projectId, pipelineId, buildId, param) "KUBERNETES" == jobPoolType -> KubernetesExecutor.getLogs(param) + "PUBLIC_DEVCLOUD" == jobPoolType -> DevCloudExecutor.getLogs(param) else -> ThirdPartExecutor.getLogs(param) } } diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/DevCloudTask.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/DevCloudTask.kt new file mode 100644 index 0000000..f681838 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/DevCloudTask.kt @@ -0,0 +1,6 @@ +package com.tencent.bk.devops.plugin.docker.pojo + +data class DevCloudTask( + val taskId: Int, + val jobName: String +) diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobParam.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobParam.kt new file mode 100644 index 0000000..f1afee1 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobParam.kt @@ -0,0 +1,20 @@ +package com.tencent.bk.devops.plugin.docker.pojo.job.request + +import com.fasterxml.jackson.annotation.JsonInclude + +data class JobParam( + @JsonInclude(JsonInclude.Include.NON_NULL) + var env: Map? = null, + val command: List? = null, + @JsonInclude(JsonInclude.Include.NON_NULL) + var nfsVolume: List? = null, + var workDir: String? = "/data/landun/workspace", + var labels: Map? = emptyMap(), + var ipEnabled: Boolean? = true +) { + data class NfsVolume( + val server: String? = null, + val path: String? = null, + val mountPath: String? = null + ) +} diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobRequest.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobRequest.kt new file mode 100644 index 0000000..10346d5 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/JobRequest.kt @@ -0,0 +1,14 @@ +package com.tencent.bk.devops.plugin.docker.pojo.job.request + +data class JobRequest( + val alias: String? = null, + val regionId: String, + val clusterType: String? = null, + val activeDeadlineSeconds: Int? = null, + val image: String? = null, + val registry: Registry? = null, + val cpu: Int? = null, + val memory: String? = null, + val params: JobParam? = null, + val podNameSelector: String? = null +) diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/PcgJobRequest.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/PcgJobRequest.kt new file mode 100644 index 0000000..67972d6 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/PcgJobRequest.kt @@ -0,0 +1,16 @@ +package com.tencent.bk.devops.plugin.docker.pojo.job.request + + +data class PcgJobRequest( + val alias: String? = null, + val regionId: String, + val clusterType: String? = null, + val activeDeadlineSeconds: Int? = null, + val image: String? = null, + val registry: Registry? = null, + val cpu: Int? = null, + val memory: String? = null, + val params: JobParam? = null, + val podNameSelector: String? = null, + val operator: String? = null +) diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/Registry.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/Registry.kt new file mode 100644 index 0000000..d141a15 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/request/Registry.kt @@ -0,0 +1,7 @@ +package com.tencent.bk.devops.plugin.docker.pojo.job.request + +data class Registry ( + val host: String, + val username: String?, + val password: String? +) \ No newline at end of file diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/response/JobResponse.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/response/JobResponse.kt new file mode 100644 index 0000000..f5a7d34 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/job/response/JobResponse.kt @@ -0,0 +1,12 @@ +package com.tencent.bk.devops.plugin.docker.pojo.job.response + +data class JobResponse ( + val actionCode: Int, + val actionMessage: String, + val data: JobResponseData +) { + data class JobResponseData ( + val name: String, + val taskId: Int + ) +} \ No newline at end of file diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusData.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusData.kt new file mode 100644 index 0000000..e334874 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusData.kt @@ -0,0 +1,22 @@ +package com.tencent.bk.devops.plugin.docker.pojo.status + +data class JobStatusData( + val deleted: Boolean, + val status: String, + val pod_result: List? +) { + data class PodResult( + val ip: String?, + val events: List? + ) + + data class PodResultEvent( + val message: String, + val reason: String, + val type: String + ) { + override fun toString(): String { + return "reason: $reason, type: $type" + } + } +} diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusResponse.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusResponse.kt new file mode 100644 index 0000000..cf77458 --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/pojo/status/JobStatusResponse.kt @@ -0,0 +1,8 @@ +package com.tencent.bk.devops.plugin.docker.pojo.status + + +data class JobStatusResponse ( + val actionCode: Int, + val actionMessage: String, + val data: JobStatusData +) \ No newline at end of file diff --git a/src/main/kotlin/com/tencent/bk/devops/plugin/docker/utils/DevCloudClient.kt b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/utils/DevCloudClient.kt new file mode 100644 index 0000000..360fbaf --- /dev/null +++ b/src/main/kotlin/com/tencent/bk/devops/plugin/docker/utils/DevCloudClient.kt @@ -0,0 +1,164 @@ +package com.tencent.bk.devops.plugin.docker.utils + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.bk.devops.plugin.docker.pojo.DevCloudTask +import com.tencent.bk.devops.plugin.docker.pojo.TaskStatus +import com.tencent.bk.devops.plugin.docker.pojo.job.request.JobRequest +import com.tencent.bk.devops.plugin.docker.pojo.job.response.JobResponse +import com.tencent.bk.devops.plugin.docker.pojo.status.JobStatusResponse +import com.tencent.bk.devops.plugin.utils.JsonUtil +import com.tencent.bk.devops.plugin.utils.OkhttpUtils +import okhttp3.* +import okhttp3.Headers.Companion.toHeaders +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.lang3.RandomStringUtils +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.* + +class DevCloudClient( + private val executeUser: String, + private val devCloudAppId: String, + private val devCloudUrl: String, + private val devCloudToken: String +) { + + companion object { + private val logger = LoggerFactory.getLogger(DevCloudClient::class.java) + } + + private fun getHeaders( + appId: String, + token: String, + staffName: String + ): Map { + val headerBuilder = mutableMapOf() + headerBuilder["APPID"] = appId + val random = RandomStringUtils.randomAlphabetic(8) + headerBuilder["RANDOM"] = random + val timestamp = (System.currentTimeMillis() / 1000).toString() + headerBuilder["TIMESTP"] = timestamp + headerBuilder["STAFFNAME"] = staffName + val encKey = DigestUtils.md5Hex(token + timestamp + random) + headerBuilder["ENCKEY"] = encKey + return headerBuilder + } + + fun createJob( + jobReq: JobRequest + ): DevCloudTask { + logger.info("start to create job: ${jobReq.alias}, ${jobReq.clusterType}, ${jobReq.regionId}, ${jobReq.params}, ${jobReq.podNameSelector}") + + val url = "$devCloudUrl/api/v2.1/job" + val body = JsonUtil.toJson(jobReq) + val request = Request.Builder().url(url) + .headers(getHeaders(devCloudAppId, devCloudToken, executeUser).toHeaders()) + .post(RequestBody.create("application/json; charset=utf-8".toMediaTypeOrNull(), body)).build() + val responseBody = OkhttpUtils.doShortHttp(request).body!!.string() + logger.info("[create job] $responseBody") + val jobRep = JsonUtil.to(responseBody, JobResponse::class.java) + if (jobRep.actionCode == 200) { + return DevCloudTask( + jobRep.data.taskId, + jobRep.data.name + ) + } else { + throw RuntimeException("create job fail") + } + } + + fun getTaskStatus( + taskId: Int + ): TaskStatus { + var countFailed = 0 + while (true) { + if (countFailed > 3) { + logger.info("Request DevCloud failed 3 times, exit with exception") + throw RuntimeException("Request DevCloud failed 3 times, exit with exception") + } + try { + val url = "$devCloudUrl/api/v2.1/tasks/$taskId" + logger.info("get task status url: $url") + val request = Request.Builder().url(url) + .headers(getHeaders(devCloudAppId, devCloudToken, executeUser).toHeaders()).get().build() + val responseBody = OkhttpUtils.doShortHttp(request).body!!.string() + logger.info("get task status response: $responseBody") + val responseMap = JsonUtil.to(responseBody, object : TypeReference>() {}) + if (responseMap["actionCode"] as? Int != 200) { + throw RuntimeException("get task status fail: $responseBody") + } + val data = responseMap["data"] as Map + return TaskStatus(status = data["status"] as String?, taskId = data["taskId"] as String?, responseBody = responseBody) + } catch (e: IOException) { + logger.info("Get DevCloud task status exception: ${e.message}") + countFailed++ + } + } + } + + fun getJobStatus( + jobName: String + ): JobStatusResponse { + var countFailed = 0 + while (true) { + if (countFailed > 3) { + logger.info("Request DevCloud failed 3 times, exit with exception") + throw RuntimeException("Request DevCloud failed 3 times, exit with exception") + } + try { + val url = "$devCloudUrl/api/v2.1/job/$jobName/status" + val request = Request.Builder().url(url) + .headers(getHeaders(devCloudAppId, devCloudToken, executeUser).toHeaders()).get().build() + val response: Response = OkhttpUtils.doShortHttp(request) + val body = response.body!!.string() + val jobStatusRep = JsonUtil.to(body, JobStatusResponse::class.java) + val actionCode: Int = jobStatusRep.actionCode + if (actionCode != 200) { + throw RuntimeException("get job status fail: $url, $body") + } + return jobStatusRep + } catch (e: IOException) { + logger.info("Get DevCloud job status exception: ${e.message}") + countFailed++ + } + } + } + + fun getLog( + jobName: String, + sinceTime: String + ): String? { + var countFailed = 0 + while (true) { + try { + val sendUrl = "$devCloudUrl/api/v2.1/job/$jobName/logs?sinceTime=$sinceTime" + val request = Request.Builder().url(sendUrl) + .headers(getHeaders(devCloudAppId, devCloudToken, executeUser).toHeaders()).get().build() + val response = OkhttpUtils.doShortHttp(request) + val res = response.body!!.string() + if (!response.isSuccessful) { + return null + } + val resultMap: Map = JsonUtil.to(res, object : TypeReference>() {}) + val data = resultMap["data"] as Map? + val logs = data?.values?.joinToString("\n") + logger.info(logs) + return logs + } catch (e: IOException) { + logger.info("Get DevCloud job log exception: ${e.message}") + if ("timeout" == e.message) { + countFailed++ + if (countFailed > 3) { + logger.info("Request DevCloud get log failed 3 times, exit with exception") + return null + } + Thread.sleep(1000L) + } else { + throw RuntimeException("Request DevCloud get log failed 3 times, exit with exception") + } + } + } + } +}