support dynamic templates for pipelines

README.md

@@ -25,3 +25,63 @@ Note that, in each case, the certificates:
 
 * Must be files reachable by the Jenkins controller.
 * Must be in the [PKCS12 format](https://en.wikipedia.org/wiki/PKCS_12).
+
+## Pipeline support
+For pipelines, nodes can be started by using the label which is assigned to a predefined job template.
+```
+node ('my-label') {
+}
+```
+Or the template can be provided directly (either in JSON or HCL).
+```
+nomad (jobTemplate:'''
+job "%WORKER_NAME%" {
+  region = "global"
+  type = "batch"
+  datacenters = ["dc1"]
+  group "jenkins-worker-taskgroup" {
+    count = 1
+    restart {
+      attempts = 0
+      mode = "fail"
+    }
+    task "jenkins-worker" {
+      driver = "docker"
+      config {
+        image = "jenkins/inbound-agent"
+        force_pull = true
+      }
+      env {
+        JENKINS_URL = "https://jenkins.service.consul"
+        JENKINS_AGENT_NAME = "%WORKER_NAME%"
+        JENKINS_SECRET = "%WORKER_SECRET%"
+        JENKINS_WEB_SOCKET = "true"
+      }
+      resources {
+        cpu = 500
+        memory = 256
+      }
+    }
+  }
+}
+''') {
+  node (JOB_LABEL) {
+    sh 'hostname'
+  }
+}
+```
+Note: The `JOB_LABEL` environment variable is defined by the `nomad` pipeline step with the corresponding label of the `dynamic template`.   
+
+Via `readFile/readTrusted`, the `jobTemplate` can also be referenced from the file system or from the repository.
+```
+nomad (jobTemplate: readTrusted('src/test/resources/nomad-templates/build.yaml')) {
+  node (JOB_LABEL) {
+    sh 'hostname'
+  }
+}
+```
+
+Optional parameters:
+* `cloud` - Name of the Nomad cloud configuration you want to use (Default: `null` which means the first Nomad cloud is selected automatically)
+* `remoteFS` - Absolute path to the working directory of the remote agent (Default: `null` which means the environment variable `hudson.model.slave.workspaceDir` or `JENKINS_HOME` is used instead)
+* `prefix` - Node name is prefixed with this value (Default: `jenkins-dt`)

pom.xml

@@ -77,6 +77,18 @@
         </pluginRepository>
     </pluginRepositories>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.jenkins.tools.bom</groupId>
+                <artifactId>bom-2.289.x</artifactId>
+                <version>987.v4ade2e49fe70</version>
+                <scope>import</scope>
+                <type>pom</type>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
         <dependency>
             <groupId>com.squareup.okhttp3</groupId>
@@ -96,12 +108,14 @@
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>credentials</artifactId>
-            <version>2.6.1</version>
         </dependency>
         <dependency>
             <groupId>org.jenkins-ci.plugins</groupId>
             <artifactId>plain-credentials</artifactId>
-            <version>1.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins.workflow</groupId>
+            <artifactId>workflow-step-api</artifactId>
         </dependency>
         <!-- TESTS -->
         <dependency>
@@ -131,5 +145,20 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins.workflow</groupId>
+            <artifactId>workflow-job</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins.workflow</groupId>
+            <artifactId>workflow-cps</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins.workflow</groupId>
+            <artifactId>workflow-basic-steps</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

src/main/java/org/jenkinsci/plugins/nomad/NomadCloud.java

@@ -20,6 +20,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Stream;
 
 import org.jenkinsci.plugins.nomad.Api.JobInfo;
 import org.jenkinsci.plugins.plaincredentials.StringCredentials;
@@ -65,6 +66,7 @@ public class NomadCloud extends AbstractCloudImpl {
     // non persistent fields
     private transient NomadApi nomad;
     private transient int pending = 0;
+    private transient List<NomadWorkerTemplate> dynamicTemplates;
 
     // legacy fields (we have to keep them for backward compatibility)
     private transient String jenkinsUrl;
@@ -96,6 +98,7 @@ public NomadCloud(
         this.serverPassword = serverPassword;
         this.prune = prune;
         this.templates = Optional.ofNullable(templates).orElse(new ArrayList<>());
+        this.dynamicTemplates = new ArrayList<>();
 
         readResolve();
     }
@@ -172,15 +175,10 @@ private void pruneOrphanedWorkers(NomadWorkerTemplate template) {
 
     // Find the correct template for job
     public NomadWorkerTemplate getTemplate(Label label) {
-        for (NomadWorkerTemplate t : templates) {
-            if (label == null && !t.getLabels().isEmpty()) {
-                continue;
-            }
-            if ((label == null && t.getLabels().isEmpty()) || (label != null && label.matches(Label.parse(t.getLabels())))) {
-                return t;
-            }
-        }
-        return null;
+        return Stream.concat(templates.stream(), dynamicTemplates.stream())
+                .filter(t -> label == null && t.getLabels().isEmpty() || (label != null && label.matches(Label.parse(t.getLabels()))))
+                .findFirst()
+                .orElse(null);
     }
 
     @Override
@@ -249,6 +247,22 @@ public Secret getServerPassword() {
         return serverPassword;
     }
 
+    /**
+     * Adds a so called 'dynamic jobTemplate' to this cloud. It is pretty much the same as the jobTemplate defined via the cloud
+     * management, but it is not editable and not visible. It can only be accessed via the assigned label.
+     */
+    public void addDynamicTemplate(NomadWorkerTemplate template) {
+        dynamicTemplates.add(template);
+    }
+
+    /**
+     * Removes an existing so called 'dynamic jobTemplate'.
+     * @param label the label assigned to the 'dynamic jobTemplate' you want to remove
+     */
+    public void removeDynamicTemplate(String label) {
+        dynamicTemplates.removeIf(t -> t.getLabels().equals(label));
+    }
+
     @Extension
     public static final class DescriptorImpl extends Descriptor<Cloud> {
 

src/main/java/org/jenkinsci/plugins/nomad/pipeline/NomadPipelineStep.java

@@ -0,0 +1,100 @@
+package org.jenkinsci.plugins.nomad.pipeline;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Set;
+
+import org.jenkinsci.plugins.workflow.steps.Step;
+import org.jenkinsci.plugins.workflow.steps.StepContext;
+import org.jenkinsci.plugins.workflow.steps.StepDescriptor;
+import org.jenkinsci.plugins.workflow.steps.StepExecution;
+import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.DataBoundSetter;
+
+import hudson.Extension;
+import hudson.model.TaskListener;
+
+/**
+ * Provides the 'nomad' method for scripted pipelines.<br>
+ * Parameters:<br>
+ * <ul>
+ *     <li>jobTemplate - Nomad Job Template (required, either in JSON or HCL)</li>
+ *     <li>cloud - Name of the Nomad cloud configuration you want to use (Default: null, which means the first cloud is used)</li>
+ *     <li>prefix - Node name is prefixed with this value (Default: jenkins-dt)</li>
+ *     <li>remoteFS - Absolute path to the working directory of the remote agent (Default: null, which means hudson.model.slave.workspaceDir
+ *     or JENKINS_HOME is used)</li>
+ * </ul>
+ */
+public class NomadPipelineStep extends Step implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final String jobTemplate;
+    private String cloud;
+    private String prefix;
+    private String remoteFS;
+
+    @DataBoundConstructor
+    public NomadPipelineStep(String jobTemplate) {
+        this.jobTemplate = jobTemplate;
+        this.prefix = "jenkins-dt";
+    }
+
+    public String getJobTemplate() {
+        return jobTemplate;
+    }
+
+    public String getCloud() {
+        return cloud;
+    }
+
+    public String getRemoteFS() {
+        return remoteFS;
+    }
+
+    public String getPrefix() {
+        return prefix;
+    }
+
+    // === optional parameters start ===
+
+    @DataBoundSetter
+    public void setCloud(String cloud) {
+        this.cloud = cloud;
+    }
+
+    @DataBoundSetter
+    public void setPrefix(String prefix) {
+        this.prefix = prefix;
+    }
+
+    @DataBoundSetter
+    public void setRemoteFS(String remoteFS) {
+        this.remoteFS = remoteFS;
+    }
+
+    // === optional parameters end ===
+
+    @Override
+    public StepExecution start(StepContext stepContext) {
+        return new NomadStepExecution(stepContext, this);
+    }
+
+    @Extension
+    public static final class DescriptorImpl extends StepDescriptor {
+        @Override
+        public String getFunctionName() {
+            return "nomad";
+        }
+
+        @Override
+        public Set<? extends Class<?>> getRequiredContext() {
+            return Collections.singleton(TaskListener.class);
+        }
+
+        @Override
+        public boolean takesImplicitBlockArgument() {
+            return true;
+        }
+    }
+}

src/main/java/org/jenkinsci/plugins/nomad/pipeline/NomadStepExecution.java

@@ -0,0 +1,88 @@
+package org.jenkinsci.plugins.nomad.pipeline;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.jenkinsci.plugins.nomad.NomadCloud;
+import org.jenkinsci.plugins.nomad.NomadWorkerTemplate;
+import org.jenkinsci.plugins.workflow.steps.BodyExecutionCallback;
+import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander;
+import org.jenkinsci.plugins.workflow.steps.StepContext;
+import org.jenkinsci.plugins.workflow.steps.StepExecution;
+
+import hudson.AbortException;
+import jenkins.model.Jenkins;
+
+/**
+ * Executes a given 'nomad' step for scripted pipelines. How it works:<br>
+ * <ol>
+ *     <li>Resolve nomad cloud by either the specified name or use the first Nomad cloud</li>
+ *     <li>Add the given job template with the cloud and assign a unique label to that template</li>
+ *     <li>Define the environment variable JOB_LABEL with the corresponding label</li>
+ *     <li>Execute the nomad step</li>
+ *     <li>Remove the job template from the cloud</li>
+ * </ol>
+ * Note: The job templates added by this step are not editable, and they are also not visible via cloud management.
+ */
+public class NomadStepExecution extends StepExecution {
+
+    private final NomadPipelineStep step;
+
+    public NomadStepExecution(StepContext stepContext, NomadPipelineStep step) {
+        super(stepContext);
+        this.step = step;
+    }
+
+    @Override
+    public boolean start() throws IOException, InterruptedException {
+        NomadCloud cloud = resolveCloud();
+        NomadWorkerTemplate template = new NomadWorkerTemplate(
+                step.getPrefix(),
+                UUID.randomUUID().toString(),
+                0,
+                false,
+                1,
+                step.getRemoteFS(),
+                step.getJobTemplate()
+        );
+
+        cloud.addDynamicTemplate(template);
+
+        getContext().newBodyInvoker()
+                .withCallback(new RemoveTemplateCallBack(template.getLabels()))
+                .withContext(EnvironmentExpander.merge(
+                        getContext().get(EnvironmentExpander.class),
+                        EnvironmentExpander.constant(Collections.singletonMap("JOB_LABEL", template.getLabels()))
+                ))
+                .start();
+
+        return false;
+    }
+
+    private NomadCloud resolveCloud() throws AbortException {
+        if (step.getCloud() == null) {
+            return Optional.ofNullable(Jenkins.get().clouds.get(NomadCloud.class))
+                    .orElseThrow(() -> new AbortException("No Nomad cloud was found. Please configure at least one!"));
+        }
+        return Optional.ofNullable(Jenkins.get().getCloud(step.getCloud()))
+                .filter(cloud -> cloud instanceof NomadCloud)
+                .map(cloud -> (NomadCloud) cloud)
+                .orElseThrow(() -> new AbortException(String.format("Nomad cloud does not exist: %s", step.getCloud())));
+    }
+
+    private class RemoveTemplateCallBack extends BodyExecutionCallback.TailCall {
+        private final String label;
+
+        public RemoveTemplateCallBack(String label) {
+            this.label = label;
+        }
+
+        @Override
+        protected void finished(StepContext stepContext) throws AbortException {
+            NomadCloud cloud = resolveCloud();
+            cloud.removeDynamicTemplate(label);
+        }
+    }
+}