diff --git a/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java b/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java index 514a839b..8d9e0f1b 100644 --- a/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java +++ b/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java @@ -94,6 +94,24 @@ public Map copyAdditionalInformation() } + @Override + public String toString() + { + try + { + return toJson(); + } + catch (JsonProcessingException e) + { + return "DataSet{" + + "serviceName='" + serviceName + '\'' + + ", commitId='" + commitId + '\'' + + ", additionalInformation=" + additionalInformation + + '}'; + } + } + + public String toJson() throws JsonProcessingException { ObjectMapper jsonMapper = new ObjectMapper(); diff --git a/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java b/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java index 92ee776b..8812c59c 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java +++ b/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java @@ -39,4 +39,4 @@ public ResponseEntity build(@Valid @RequestBody BuildRequest request) pipelineService.publish(request); return ResponseEntity.ok().build(); } -} \ No newline at end of file +} diff --git a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java index 0ebe9963..e312b6ca 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java +++ b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java @@ -1,21 +1,30 @@ package com.freenow.sauron.properties; import java.util.Collections; -import java.util.HashMap; import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; -@ConfigurationProperties("sauron.pipelines") -public class PipelineConfigurationProperties extends HashMap> +@Getter +@Setter +@ConfigurationProperties(prefix = "sauron") +public class PipelineConfigurationProperties { + //A map of pipeline names to the list of plugin IDs that comprise them.Spring Boot will bind properties like `sauron.pipelines.default` into this map. + private Map> pipelines = Collections.emptyMap(); + + //The ID of the plugin that must be executed at the end of a user-defined pipeline run. + private String mandatoryOutputPlugin = "elasticsearch-output"; + public List getDefaultPipeline() { return getPipeline("default"); } - public List getPipeline(String pipeline) { - return this.getOrDefault(pipeline, Collections.emptyList()); + return pipelines.getOrDefault(pipeline, Collections.emptyList()); } } diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 7428a3bf..9febb90b 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -7,6 +7,8 @@ import com.freenow.sauron.plugins.SauronExtension; import com.freenow.sauron.properties.PipelineConfigurationProperties; import com.freenow.sauron.properties.PluginsConfigurationProperties; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -21,8 +23,6 @@ @EnableConfigurationProperties({PipelineConfigurationProperties.class, PluginsConfigurationProperties.class}) public class PipelineService { - private static final String ELASTICSEARCH_OUTPUT_PLUGIN = "elasticsearch-output"; - private final PipelineConfigurationProperties pipelineProperties; private final PluginsConfigurationProperties pluginsProperties; @@ -31,18 +31,22 @@ public class PipelineService private final RequestHandler handler; + private final MeterRegistry meterRegistry; + @Autowired public PipelineService( PluginManager pluginManager, PipelineConfigurationProperties pipelineProperties, PluginsConfigurationProperties pluginsProperties, - RequestHandler handler) + RequestHandler handler, + MeterRegistry meterRegistry) { this.pluginManager = pluginManager; this.pipelineProperties = pipelineProperties; this.pluginsProperties = pluginsProperties; this.handler = handler; + this.meterRegistry = meterRegistry; handler.setConsumer(this::process); } @@ -51,6 +55,7 @@ public void publish(BuildRequest request) { try { + log.info("Received request to publish: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); handler.handle(request); } catch (Exception ex) @@ -64,7 +69,9 @@ void process(BuildRequest request) { try { - final DataSet dataSet = BuildMapper.makeDataSet(request); + log.info("Starting processing for request: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); + DataSet dataSet = BuildMapper.makeDataSet(request); + log.debug("Initial DataSet created from request: {}", dataSet); String plugin = request.getPlugin(); if (StringUtils.isNotBlank(plugin)) @@ -72,60 +79,93 @@ void process(BuildRequest request) plugin = StringUtils.lowerCase(request.getPlugin()); final List defaultPipeline = pipelineProperties.getDefaultPipeline(); - log.debug("Running user defined pipeline."); + log.debug("User-defined plugin specified: {}. Running user defined pipeline. Default pipeline plugins: {}", plugin, defaultPipeline); if (defaultPipeline.contains(plugin)) { - runDependencies(request, dataSet, plugin, defaultPipeline); + log.debug("User-defined plugin '{}' is part of the default pipeline. Running dependencies first.", plugin); + runDependencies(dataSet, plugin, defaultPipeline); } - runPlugin(plugin, request, dataSet); - runPlugin(ELASTICSEARCH_OUTPUT_PLUGIN, request, dataSet); + log.debug("Executing user-defined plugin: {}", plugin); + runPlugin(plugin, dataSet); + + String mandatoryOutputPlugin = pipelineProperties.getMandatoryOutputPlugin(); + if (StringUtils.isNotBlank(mandatoryOutputPlugin)) { + log.debug("Executing mandatory output plugin: {}", mandatoryOutputPlugin); + runPlugin(mandatoryOutputPlugin, dataSet); + } } else { - log.debug("Running default pipeline."); - pipelineProperties.getDefaultPipeline().forEach(pluginId -> runPlugin(pluginId, request, dataSet)); + log.debug("No user-defined plugin. Running default pipeline. Default pipeline plugins: {}", pipelineProperties.getDefaultPipeline()); + pipelineProperties.getDefaultPipeline().forEach(pluginId -> runPlugin(pluginId, dataSet)); } } catch (final Exception ex) { - log.error(String.format("Error loading plugins: %s", ex.getMessage()), ex); + log.error("Error processing request for serviceName={}, commitId={}", request.getServiceName(), request.getCommitId(), ex); } } private void runDependencies( - final BuildRequest request, final DataSet dataSet, - final String plugin, final List defaultPipeline) + DataSet dataSet, final String plugin, final List defaultPipeline) { for (final String defaultPipelinePlugin : defaultPipeline) { if (StringUtils.equals(plugin, defaultPipelinePlugin)) { + log.debug("Dependency plugin '{}' is the main plugin '{}', skipping further dependencies.", defaultPipelinePlugin, plugin); return; } - runPlugin(defaultPipelinePlugin, request, dataSet); + log.debug("Running dependency plugin: {} for main plugin: {}", defaultPipelinePlugin, plugin); + runPlugin(defaultPipelinePlugin, dataSet); } } - void runPlugin(String plugin, BuildRequest request, DataSet dataSet) + void runPlugin(String plugin, DataSet dataSet) { - pluginManager.getExtensions(SauronExtension.class, plugin).forEach(pluginExtension -> { + for (SauronExtension pluginExtension : pluginManager.getExtensions(SauronExtension.class, plugin)) + { try { - log.debug(String.format("Applying pluginId: %s. Processing service %s - %s", plugin, request.getServiceName(), request.getCommitId())); MDC.put("sauron.pluginId", plugin); - MDC.put("sauron.serviceName", request.getServiceName()); - MDC.put("sauron.commitId", request.getCommitId()); - pluginExtension.apply(pluginsProperties, dataSet); + MDC.put("sauron.serviceName", dataSet.getServiceName()); + MDC.put("sauron.commitId", dataSet.getCommitId()); + MDC.put("sauron.buildId", dataSet.getBuildId()); + log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), + dataSet); + + getTimerBuilder("sauron.plugin.execution.time") + .tag("plugin", plugin) + .tag("service", dataSet.getServiceName()) + .tag("commit", dataSet.getCommitId()) + .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSet)); + + meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "success").increment(); + log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet); } catch (final Exception ex) { - log.error(String.format("Error processing pipeline: %s:%s. %s", request.getServiceName(), request.getCommitId(), ex.getMessage()), ex); + meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "failure").increment(); + log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet, ex); + } + finally + { + MDC.remove("sauron.pluginId"); + MDC.remove("sauron.serviceName"); + MDC.remove("sauron.commitId"); + MDC.remove("sauron.buildId"); } - }); + } + } + + + Timer.Builder getTimerBuilder(String name) + { + return Timer.builder(name); } } diff --git a/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java b/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java index b0eb8f0d..fa46c8a7 100644 --- a/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java +++ b/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java @@ -6,11 +6,16 @@ import com.freenow.sauron.plugins.SauronExtension; import com.freenow.sauron.properties.PipelineConfigurationProperties; import com.freenow.sauron.properties.PluginsConfigurationProperties; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.util.Arrays; import java.util.Collections; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -19,15 +24,19 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.mockito.internal.verification.VerificationModeFactory.times; -@RunWith(MockitoJUnitRunner.Silent.class) -public class PipelineServiceTest extends UtilsBaseTest + +@RunWith(MockitoJUnitRunner.class) +public class PipelineServiceTest { private static final String ELASTICSEARCH_OUTPUT_PLUGIN = "elasticsearch-output"; @@ -49,26 +58,52 @@ public class PipelineServiceTest extends UtilsBaseTest @Spy private RequestHandler requestHandler; - @Spy - @InjectMocks + @Mock + private MeterRegistry meterRegistry; + + @Mock + private Counter counter; + + @Mock + private Timer.Builder timerBuilder; + + @Mock + private Timer timer; + private PipelineService pipelineService; - @Test - public void testProcessEmptyPipeline() + @Before + public void setup() { - doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); - pipelineService.process(new BuildRequest()); - verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), anyString()); + // Initialize the spy here + pipelineService = spy(new PipelineService( + pluginManager, + pipelineProperties, + pluginsProperties, + requestHandler, + meterRegistry + )); + + when(pipelineProperties.getMandatoryOutputPlugin()).thenReturn(ELASTICSEARCH_OUTPUT_PLUGIN); + + when(pluginManager.getExtensions(eq(SauronExtension.class), anyString())).thenReturn(Collections.singletonList(extension)); + + when(meterRegistry.counter(anyString(), anyString(), anyString(), anyString(), anyString())).thenReturn(counter); + + doReturn(timerBuilder).when(pipelineService).getTimerBuilder(anyString()); + when(timerBuilder.tag(anyString(), anyString())).thenReturn(timerBuilder); + when(timerBuilder.register(any(MeterRegistry.class))).thenReturn(timer); + doAnswer(invocation -> ((Supplier) invocation.getArgument(0)).get()) + .when(timer).record(any(Supplier.class)); } - @Test - public void testProcessDefaultPipeline() + public void testProcessEmptyPipeline() { - doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - pipelineService.process(new BuildRequest()); - verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); + doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); + pipelineService.process(buildDefaultRequest()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), anyString()); } @@ -76,10 +111,10 @@ public void testProcessDefaultPipeline() public void testProcessDefaultPipelineExistingPlugin() { doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - doReturn(Collections.singletonList(extension)).when(pluginManager).getExtensions(eq(SauronExtension.class), eq("plugin")); - pipelineService.process(new BuildRequest()); - verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); - verify(extension, times(1)).apply(eq(pluginsProperties), any(DataSet.class)); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); + pipelineService.process(buildDefaultRequest()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("plugin")); + verify(extension, times(1)).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -87,8 +122,8 @@ public void testProcessDefaultPipelineExistingPlugin() public void testProcessDefaultPipelineNotExistingPlugin() { doReturn(Collections.singletonList("invalidPlugin")).when(pipelineProperties).getDefaultPipeline(); - doReturn(Collections.singletonList(extension)).when(pluginManager).getExtensions(eq(SauronExtension.class), eq("plugin")); - pipelineService.process(new BuildRequest()); + when(pluginManager.getExtensions(eq(SauronExtension.class), eq("invalidPlugin"))).thenReturn(Collections.emptyList()); + pipelineService.process(buildDefaultRequest()); verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); verify(extension, never()).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -98,8 +133,8 @@ public void testProcessDefaultPipelineNotExistingPlugin() public void testProcessExceptionThrown() { doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - doThrow(RuntimeException.class).when(pluginManager).getExtensions(any(), anyString()); - pipelineService.process(new BuildRequest()); + doThrow(RuntimeException.class).when(pluginManager).getExtensions(any(), eq("plugin")); + pipelineService.process(buildDefaultRequest()); verify(pluginManager, times(1)).getExtensions(any(), anyString()); verify(extension, never()).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -109,11 +144,12 @@ public void testProcessExceptionThrown() public void pluginShouldRunWhenDefaultPipelineIsEmpty() { doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -121,12 +157,13 @@ public void pluginShouldRunWhenDefaultPipelineIsEmpty() public void pluginShouldRunWhenNotPresentInDefaultPipeline() { doReturn(Collections.singletonList("random-plugin")).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, times(0)).runPlugin(eq("random-plugin"), any(), any()); - verify(pipelineService).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), eq("random-plugin")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -134,11 +171,12 @@ public void pluginShouldRunWhenNotPresentInDefaultPipeline() public void pluginShouldRunOnlyOnceWhenPresentInDefaultPipeline() { doReturn(Collections.singletonList(REPROCESS_PLUGIN)).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, atMost(1)).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -146,22 +184,31 @@ public void pluginShouldRunOnlyOnceWhenPresentInDefaultPipeline() public void pluginDependenciesShouldRun() { doReturn(Arrays.asList("dependency-1", "dependency-2", REPROCESS_PLUGIN, "another-plugin")).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, times(0)).runPlugin(eq("another-plugin"), any(), any()); - verify(pipelineService).runPlugin(eq("dependency-1"), any(), any()); - verify(pipelineService).runPlugin(eq("dependency-2"), any(), any()); - verify(pipelineService, atMost(1)).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), eq("another-plugin")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("dependency-1")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("dependency-2")); + verify(pluginManager, atLeastOnce()).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + } + + private BuildRequest buildDefaultRequest() + { + final BuildRequest request = new BuildRequest(); + request.setServiceName("test-service"); + request.setCommitId("abc1234"); + request.setBuildId(UUID.randomUUID().toString()); + return request; } private BuildRequest buildReprocessRequest() { - final BuildRequest request = new BuildRequest(); + final BuildRequest request = buildDefaultRequest(); request.setPlugin(REPROCESS_PLUGIN); return request; } -} \ No newline at end of file +} diff --git a/sauron-service/src/test/resources/logback-test.xml b/sauron-service/src/test/resources/logback-test.xml index ee0ec506..65bbfb39 100644 --- a/sauron-service/src/test/resources/logback-test.xml +++ b/sauron-service/src/test/resources/logback-test.xml @@ -1,9 +1,7 @@ - - + value="%date{ISO8601} | %-16thread | %-5level{5} | [%X{sauron.serviceName:-unknown-service}] [%X{sauron.pluginId:-no-plugin}] | %-150message | %-35(%logger{0}:%-5L)%n%exception{full}"/> UTF-8 @@ -14,4 +12,4 @@ - \ No newline at end of file +