From 69ec27828784017428a971c88e7ac69157e8871b Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Mon, 27 Oct 2025 09:06:19 +0100 Subject: [PATCH 1/7] parameterized logging defined --- .../com/freenow/sauron/model/DataSet.java | 18 +++++++++++++ .../sauron/service/PipelineService.java | 25 +++++++++++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java b/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java index 514a839b..8d9e0f1b 100644 --- a/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java +++ b/sauron-core/src/main/java/com/freenow/sauron/model/DataSet.java @@ -94,6 +94,24 @@ public Map copyAdditionalInformation() } + @Override + public String toString() + { + try + { + return toJson(); + } + catch (JsonProcessingException e) + { + return "DataSet{" + + "serviceName='" + serviceName + '\'' + + ", commitId='" + commitId + '\'' + + ", additionalInformation=" + additionalInformation + + '}'; + } + } + + public String toJson() throws JsonProcessingException { ObjectMapper jsonMapper = new ObjectMapper(); diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 7428a3bf..7de74adb 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -51,6 +51,7 @@ public void publish(BuildRequest request) { try { + log.info("Received request to publish: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); handler.handle(request); } catch (Exception ex) @@ -64,7 +65,9 @@ void process(BuildRequest request) { try { + log.info("Starting processing for request: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); final DataSet dataSet = BuildMapper.makeDataSet(request); + log.debug("Initial DataSet created from request: {}", dataSet); String plugin = request.getPlugin(); if (StringUtils.isNotBlank(plugin)) @@ -72,25 +75,28 @@ void process(BuildRequest request) plugin = StringUtils.lowerCase(request.getPlugin()); final List defaultPipeline = pipelineProperties.getDefaultPipeline(); - log.debug("Running user defined pipeline."); + log.debug("User-defined plugin specified: {}. Running user defined pipeline. Default pipeline plugins: {}", plugin, defaultPipeline); if (defaultPipeline.contains(plugin)) { + log.debug("User-defined plugin '{}' is part of the default pipeline. Running dependencies first.", plugin); runDependencies(request, dataSet, plugin, defaultPipeline); } + log.debug("Executing user-defined plugin: {}", plugin); runPlugin(plugin, request, dataSet); + log.debug("Executing mandatory output plugin: {}", ELASTICSEARCH_OUTPUT_PLUGIN); runPlugin(ELASTICSEARCH_OUTPUT_PLUGIN, request, dataSet); } else { - log.debug("Running default pipeline."); + log.debug("No user-defined plugin. Running default pipeline. Default pipeline plugins: {}", pipelineProperties.getDefaultPipeline()); pipelineProperties.getDefaultPipeline().forEach(pluginId -> runPlugin(pluginId, request, dataSet)); } } catch (final Exception ex) { - log.error(String.format("Error loading plugins: %s", ex.getMessage()), ex); + log.error("Error processing request for serviceName={}, commitId={}", request.getServiceName(), request.getCommitId(), ex); } } @@ -103,9 +109,11 @@ private void runDependencies( { if (StringUtils.equals(plugin, defaultPipelinePlugin)) { + log.debug("Dependency plugin '{}' is the main plugin '{}', skipping further dependencies.", defaultPipelinePlugin, plugin); return; } + log.debug("Running dependency plugin: {} for main plugin: {}", defaultPipelinePlugin, plugin); runPlugin(defaultPipelinePlugin, request, dataSet); } } @@ -116,15 +124,22 @@ void runPlugin(String plugin, BuildRequest request, DataSet dataSet) pluginManager.getExtensions(SauronExtension.class, plugin).forEach(pluginExtension -> { try { - log.debug(String.format("Applying pluginId: %s. Processing service %s - %s", plugin, request.getServiceName(), request.getCommitId())); MDC.put("sauron.pluginId", plugin); MDC.put("sauron.serviceName", request.getServiceName()); MDC.put("sauron.commitId", request.getCommitId()); + log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); pluginExtension.apply(pluginsProperties, dataSet); + log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); } catch (final Exception ex) { - log.error(String.format("Error processing pipeline: %s:%s. %s", request.getServiceName(), request.getCommitId(), ex.getMessage()), ex); + log.error("Error in plugin '{}' for serviceName={}, commitId={}", plugin, request.getServiceName(), request.getCommitId(), ex); + } + finally + { + MDC.remove("sauron.pluginId"); + MDC.remove("sauron.serviceName"); + MDC.remove("sauron.commitId"); } }); } From 2ba27704d8cb467b471b0288842f598c2ce9e2ca Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Mon, 27 Oct 2025 09:46:57 +0100 Subject: [PATCH 2/7] execution time logged --- .../main/java/com/freenow/sauron/service/PipelineService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 7de74adb..6a2158cc 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -128,7 +128,11 @@ void runPlugin(String plugin, BuildRequest request, DataSet dataSet) MDC.put("sauron.serviceName", request.getServiceName()); MDC.put("sauron.commitId", request.getCommitId()); log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); + + long startTime = System.currentTimeMillis(); pluginExtension.apply(pluginsProperties, dataSet); + long duration = System.currentTimeMillis() - startTime; + log.info("Plugin '{}' executed in {}ms", plugin, duration); log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); } catch (final Exception ex) From 1e1b750c500342038fe744cca393b2678032f356 Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Mon, 27 Oct 2025 10:49:10 +0100 Subject: [PATCH 3/7] sauron buildID logged --- .../main/java/com/freenow/sauron/service/PipelineService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 6a2158cc..59eae457 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -127,6 +127,7 @@ void runPlugin(String plugin, BuildRequest request, DataSet dataSet) MDC.put("sauron.pluginId", plugin); MDC.put("sauron.serviceName", request.getServiceName()); MDC.put("sauron.commitId", request.getCommitId()); + MDC.put("sauron.buildId", request.getBuildId()); log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); long startTime = System.currentTimeMillis(); @@ -137,13 +138,14 @@ void runPlugin(String plugin, BuildRequest request, DataSet dataSet) } catch (final Exception ex) { - log.error("Error in plugin '{}' for serviceName={}, commitId={}", plugin, request.getServiceName(), request.getCommitId(), ex); + log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet, ex); } finally { MDC.remove("sauron.pluginId"); MDC.remove("sauron.serviceName"); MDC.remove("sauron.commitId"); + MDC.remove("sauron.buildId"); } }); } From de9eac2bb29cd44c673f539d580f90d4754dcd0a Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Tue, 28 Oct 2025 09:43:50 +0100 Subject: [PATCH 4/7] pipeline service logging and metrics exposing updated --- .../PipelineConfigurationProperties.java | 24 +++- .../sauron/service/PipelineService.java | 69 ++++++---- .../sauron/service/PipelineServiceTest.java | 123 ++++++++++++------ .../src/test/resources/logback-test.xml | 6 +- 4 files changed, 150 insertions(+), 72 deletions(-) diff --git a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java index 0ebe9963..24ec3039 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java +++ b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java @@ -1,21 +1,35 @@ package com.freenow.sauron.properties; import java.util.Collections; -import java.util.HashMap; import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; -@ConfigurationProperties("sauron.pipelines") -public class PipelineConfigurationProperties extends HashMap> +@Getter +@Setter +@ConfigurationProperties(prefix = "sauron") +public class PipelineConfigurationProperties { + /** + * A map of pipeline names to the list of plugin IDs that comprise them. + * Spring Boot will bind properties like `sauron.pipelines.default` into this map. + */ + private Map> pipelines = Collections.emptyMap(); + + /** + * The ID of the plugin that must be executed at the end of a user-defined pipeline run. + */ + private String mandatoryOutputPlugin = "elasticsearch-output"; + public List getDefaultPipeline() { return getPipeline("default"); } - public List getPipeline(String pipeline) { - return this.getOrDefault(pipeline, Collections.emptyList()); + return pipelines.getOrDefault(pipeline, Collections.emptyList()); } } diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 59eae457..90548bfa 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -7,7 +7,10 @@ import com.freenow.sauron.plugins.SauronExtension; import com.freenow.sauron.properties.PipelineConfigurationProperties; import com.freenow.sauron.properties.PluginsConfigurationProperties; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.pf4j.PluginManager; @@ -21,8 +24,6 @@ @EnableConfigurationProperties({PipelineConfigurationProperties.class, PluginsConfigurationProperties.class}) public class PipelineService { - private static final String ELASTICSEARCH_OUTPUT_PLUGIN = "elasticsearch-output"; - private final PipelineConfigurationProperties pipelineProperties; private final PluginsConfigurationProperties pluginsProperties; @@ -31,18 +32,22 @@ public class PipelineService private final RequestHandler handler; + private final MeterRegistry meterRegistry; + @Autowired public PipelineService( PluginManager pluginManager, PipelineConfigurationProperties pipelineProperties, PluginsConfigurationProperties pluginsProperties, - RequestHandler handler) + RequestHandler handler, + MeterRegistry meterRegistry) { this.pluginManager = pluginManager; this.pipelineProperties = pipelineProperties; this.pluginsProperties = pluginsProperties; this.handler = handler; + this.meterRegistry = meterRegistry; handler.setConsumer(this::process); } @@ -66,8 +71,8 @@ void process(BuildRequest request) try { log.info("Starting processing for request: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); - final DataSet dataSet = BuildMapper.makeDataSet(request); - log.debug("Initial DataSet created from request: {}", dataSet); + final AtomicReference dataSet = new AtomicReference<>(BuildMapper.makeDataSet(request)); + log.debug("Initial DataSet created from request: {}", dataSet.get()); String plugin = request.getPlugin(); if (StringUtils.isNotBlank(plugin)) @@ -80,18 +85,22 @@ void process(BuildRequest request) if (defaultPipeline.contains(plugin)) { log.debug("User-defined plugin '{}' is part of the default pipeline. Running dependencies first.", plugin); - runDependencies(request, dataSet, plugin, defaultPipeline); + runDependencies(dataSet, plugin, defaultPipeline); } log.debug("Executing user-defined plugin: {}", plugin); - runPlugin(plugin, request, dataSet); - log.debug("Executing mandatory output plugin: {}", ELASTICSEARCH_OUTPUT_PLUGIN); - runPlugin(ELASTICSEARCH_OUTPUT_PLUGIN, request, dataSet); + runPlugin(plugin, dataSet); + + String mandatoryOutputPlugin = pipelineProperties.getMandatoryOutputPlugin(); + if (StringUtils.isNotBlank(mandatoryOutputPlugin)) { + log.debug("Executing mandatory output plugin: {}", mandatoryOutputPlugin); + runPlugin(mandatoryOutputPlugin, dataSet); + } } else { log.debug("No user-defined plugin. Running default pipeline. Default pipeline plugins: {}", pipelineProperties.getDefaultPipeline()); - pipelineProperties.getDefaultPipeline().forEach(pluginId -> runPlugin(pluginId, request, dataSet)); + pipelineProperties.getDefaultPipeline().forEach(pluginId -> runPlugin(pluginId, dataSet)); } } catch (final Exception ex) @@ -102,8 +111,7 @@ void process(BuildRequest request) private void runDependencies( - final BuildRequest request, final DataSet dataSet, - final String plugin, final List defaultPipeline) + final AtomicReference dataSet, final String plugin, final List defaultPipeline) { for (final String defaultPipelinePlugin : defaultPipeline) { @@ -114,31 +122,36 @@ private void runDependencies( } log.debug("Running dependency plugin: {} for main plugin: {}", defaultPipelinePlugin, plugin); - runPlugin(defaultPipelinePlugin, request, dataSet); + runPlugin(defaultPipelinePlugin, dataSet); } } - void runPlugin(String plugin, BuildRequest request, DataSet dataSet) + void runPlugin(String plugin, AtomicReference dataSet) { pluginManager.getExtensions(SauronExtension.class, plugin).forEach(pluginExtension -> { try { MDC.put("sauron.pluginId", plugin); - MDC.put("sauron.serviceName", request.getServiceName()); - MDC.put("sauron.commitId", request.getCommitId()); - MDC.put("sauron.buildId", request.getBuildId()); - log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); - - long startTime = System.currentTimeMillis(); - pluginExtension.apply(pluginsProperties, dataSet); - long duration = System.currentTimeMillis() - startTime; - log.info("Plugin '{}' executed in {}ms", plugin, duration); - log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet); + MDC.put("sauron.serviceName", dataSet.get().getServiceName()); + MDC.put("sauron.commitId", dataSet.get().getCommitId()); + MDC.put("sauron.buildId", dataSet.get().getBuildId()); + log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get()); + + DataSet newDataSet = getTimerBuilder("sauron.plugin.execution.time") + .tag("plugin", plugin) + .tag("service", dataSet.get().getServiceName()) + .tag("commit", dataSet.get().getCommitId()) + .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSet.get())); + dataSet.set(newDataSet); + + meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "success").increment(); + log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get()); } catch (final Exception ex) { - log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, request.getServiceName(), request.getCommitId(), dataSet, ex); + meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "failure").increment(); + log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get(), ex); } finally { @@ -149,4 +162,10 @@ void runPlugin(String plugin, BuildRequest request, DataSet dataSet) } }); } + + + Timer.Builder getTimerBuilder(String name) + { + return Timer.builder(name); + } } diff --git a/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java b/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java index b0eb8f0d..fa46c8a7 100644 --- a/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java +++ b/sauron-service/src/test/java/com/freenow/sauron/service/PipelineServiceTest.java @@ -6,11 +6,16 @@ import com.freenow.sauron.plugins.SauronExtension; import com.freenow.sauron.properties.PipelineConfigurationProperties; import com.freenow.sauron.properties.PluginsConfigurationProperties; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; import java.util.Arrays; import java.util.Collections; +import java.util.UUID; +import java.util.function.Supplier; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -19,15 +24,19 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.mockito.internal.verification.VerificationModeFactory.times; -@RunWith(MockitoJUnitRunner.Silent.class) -public class PipelineServiceTest extends UtilsBaseTest + +@RunWith(MockitoJUnitRunner.class) +public class PipelineServiceTest { private static final String ELASTICSEARCH_OUTPUT_PLUGIN = "elasticsearch-output"; @@ -49,26 +58,52 @@ public class PipelineServiceTest extends UtilsBaseTest @Spy private RequestHandler requestHandler; - @Spy - @InjectMocks + @Mock + private MeterRegistry meterRegistry; + + @Mock + private Counter counter; + + @Mock + private Timer.Builder timerBuilder; + + @Mock + private Timer timer; + private PipelineService pipelineService; - @Test - public void testProcessEmptyPipeline() + @Before + public void setup() { - doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); - pipelineService.process(new BuildRequest()); - verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), anyString()); + // Initialize the spy here + pipelineService = spy(new PipelineService( + pluginManager, + pipelineProperties, + pluginsProperties, + requestHandler, + meterRegistry + )); + + when(pipelineProperties.getMandatoryOutputPlugin()).thenReturn(ELASTICSEARCH_OUTPUT_PLUGIN); + + when(pluginManager.getExtensions(eq(SauronExtension.class), anyString())).thenReturn(Collections.singletonList(extension)); + + when(meterRegistry.counter(anyString(), anyString(), anyString(), anyString(), anyString())).thenReturn(counter); + + doReturn(timerBuilder).when(pipelineService).getTimerBuilder(anyString()); + when(timerBuilder.tag(anyString(), anyString())).thenReturn(timerBuilder); + when(timerBuilder.register(any(MeterRegistry.class))).thenReturn(timer); + doAnswer(invocation -> ((Supplier) invocation.getArgument(0)).get()) + .when(timer).record(any(Supplier.class)); } - @Test - public void testProcessDefaultPipeline() + public void testProcessEmptyPipeline() { - doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - pipelineService.process(new BuildRequest()); - verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); + doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); + pipelineService.process(buildDefaultRequest()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), anyString()); } @@ -76,10 +111,10 @@ public void testProcessDefaultPipeline() public void testProcessDefaultPipelineExistingPlugin() { doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - doReturn(Collections.singletonList(extension)).when(pluginManager).getExtensions(eq(SauronExtension.class), eq("plugin")); - pipelineService.process(new BuildRequest()); - verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); - verify(extension, times(1)).apply(eq(pluginsProperties), any(DataSet.class)); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); + pipelineService.process(buildDefaultRequest()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("plugin")); + verify(extension, times(1)).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -87,8 +122,8 @@ public void testProcessDefaultPipelineExistingPlugin() public void testProcessDefaultPipelineNotExistingPlugin() { doReturn(Collections.singletonList("invalidPlugin")).when(pipelineProperties).getDefaultPipeline(); - doReturn(Collections.singletonList(extension)).when(pluginManager).getExtensions(eq(SauronExtension.class), eq("plugin")); - pipelineService.process(new BuildRequest()); + when(pluginManager.getExtensions(eq(SauronExtension.class), eq("invalidPlugin"))).thenReturn(Collections.emptyList()); + pipelineService.process(buildDefaultRequest()); verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), anyString()); verify(extension, never()).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -98,8 +133,8 @@ public void testProcessDefaultPipelineNotExistingPlugin() public void testProcessExceptionThrown() { doReturn(Collections.singletonList("plugin")).when(pipelineProperties).getDefaultPipeline(); - doThrow(RuntimeException.class).when(pluginManager).getExtensions(any(), anyString()); - pipelineService.process(new BuildRequest()); + doThrow(RuntimeException.class).when(pluginManager).getExtensions(any(), eq("plugin")); + pipelineService.process(buildDefaultRequest()); verify(pluginManager, times(1)).getExtensions(any(), anyString()); verify(extension, never()).apply(any(PluginsConfigurationProperties.class), any(DataSet.class)); } @@ -109,11 +144,12 @@ public void testProcessExceptionThrown() public void pluginShouldRunWhenDefaultPipelineIsEmpty() { doReturn(Collections.emptyList()).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -121,12 +157,13 @@ public void pluginShouldRunWhenDefaultPipelineIsEmpty() public void pluginShouldRunWhenNotPresentInDefaultPipeline() { doReturn(Collections.singletonList("random-plugin")).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, times(0)).runPlugin(eq("random-plugin"), any(), any()); - verify(pipelineService).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), eq("random-plugin")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -134,11 +171,12 @@ public void pluginShouldRunWhenNotPresentInDefaultPipeline() public void pluginShouldRunOnlyOnceWhenPresentInDefaultPipeline() { doReturn(Collections.singletonList(REPROCESS_PLUGIN)).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, atMost(1)).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq(ELASTICSEARCH_OUTPUT_PLUGIN)); } @@ -146,22 +184,31 @@ public void pluginShouldRunOnlyOnceWhenPresentInDefaultPipeline() public void pluginDependenciesShouldRun() { doReturn(Arrays.asList("dependency-1", "dependency-2", REPROCESS_PLUGIN, "another-plugin")).when(pipelineProperties).getDefaultPipeline(); + when(extension.apply(any(PluginsConfigurationProperties.class), any(DataSet.class))).thenAnswer(invocation -> invocation.getArgument(1)); pipelineService.process(buildReprocessRequest()); - verify(pipelineService, times(0)).runPlugin(eq("another-plugin"), any(), any()); - verify(pipelineService).runPlugin(eq("dependency-1"), any(), any()); - verify(pipelineService).runPlugin(eq("dependency-2"), any(), any()); - verify(pipelineService, atMost(1)).runPlugin(eq(REPROCESS_PLUGIN), any(), any()); - verify(pipelineService).runPlugin(eq(ELASTICSEARCH_OUTPUT_PLUGIN), any(), any()); + verify(pluginManager, never()).getExtensions(eq(SauronExtension.class), eq("another-plugin")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("dependency-1")); + verify(pluginManager, times(1)).getExtensions(eq(SauronExtension.class), eq("dependency-2")); + verify(pluginManager, atLeastOnce()).getExtensions(eq(SauronExtension.class), eq(REPROCESS_PLUGIN)); + } + + private BuildRequest buildDefaultRequest() + { + final BuildRequest request = new BuildRequest(); + request.setServiceName("test-service"); + request.setCommitId("abc1234"); + request.setBuildId(UUID.randomUUID().toString()); + return request; } private BuildRequest buildReprocessRequest() { - final BuildRequest request = new BuildRequest(); + final BuildRequest request = buildDefaultRequest(); request.setPlugin(REPROCESS_PLUGIN); return request; } -} \ No newline at end of file +} diff --git a/sauron-service/src/test/resources/logback-test.xml b/sauron-service/src/test/resources/logback-test.xml index ee0ec506..65bbfb39 100644 --- a/sauron-service/src/test/resources/logback-test.xml +++ b/sauron-service/src/test/resources/logback-test.xml @@ -1,9 +1,7 @@ - - + value="%date{ISO8601} | %-16thread | %-5level{5} | [%X{sauron.serviceName:-unknown-service}] [%X{sauron.pluginId:-no-plugin}] | %-150message | %-35(%logger{0}:%-5L)%n%exception{full}"/> UTF-8 @@ -14,4 +12,4 @@ - \ No newline at end of file + From cbaeb7ee738283f9eddc214a446c1d98419b27d0 Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Tue, 28 Oct 2025 10:19:08 +0100 Subject: [PATCH 5/7] comments updated --- .../properties/PipelineConfigurationProperties.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java index 24ec3039..e312b6ca 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java +++ b/sauron-service/src/main/java/com/freenow/sauron/properties/PipelineConfigurationProperties.java @@ -12,15 +12,10 @@ @ConfigurationProperties(prefix = "sauron") public class PipelineConfigurationProperties { - /** - * A map of pipeline names to the list of plugin IDs that comprise them. - * Spring Boot will bind properties like `sauron.pipelines.default` into this map. - */ + //A map of pipeline names to the list of plugin IDs that comprise them.Spring Boot will bind properties like `sauron.pipelines.default` into this map. private Map> pipelines = Collections.emptyMap(); - /** - * The ID of the plugin that must be executed at the end of a user-defined pipeline run. - */ + //The ID of the plugin that must be executed at the end of a user-defined pipeline run. private String mandatoryOutputPlugin = "elasticsearch-output"; public List getDefaultPipeline() From 6e4449a8b3c56e34ba4c8d8d14c8cfa9cc9847f8 Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Wed, 29 Oct 2025 17:09:40 +0100 Subject: [PATCH 6/7] removed atomic reference --- .../sauron/controller/PipelineController.java | 2 +- .../sauron/service/PipelineService.java | 39 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java b/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java index 92ee776b..8812c59c 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java +++ b/sauron-service/src/main/java/com/freenow/sauron/controller/PipelineController.java @@ -39,4 +39,4 @@ public ResponseEntity build(@Valid @RequestBody BuildRequest request) pipelineService.publish(request); return ResponseEntity.ok().build(); } -} \ No newline at end of file +} diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 90548bfa..29dd1f67 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -10,7 +10,6 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.pf4j.PluginManager; @@ -71,8 +70,8 @@ void process(BuildRequest request) try { log.info("Starting processing for request: serviceName={}, commitId={}", request.getServiceName(), request.getCommitId()); - final AtomicReference dataSet = new AtomicReference<>(BuildMapper.makeDataSet(request)); - log.debug("Initial DataSet created from request: {}", dataSet.get()); + DataSet dataSet = BuildMapper.makeDataSet(request); + log.debug("Initial DataSet created from request: {}", dataSet); String plugin = request.getPlugin(); if (StringUtils.isNotBlank(plugin)) @@ -111,7 +110,7 @@ void process(BuildRequest request) private void runDependencies( - final AtomicReference dataSet, final String plugin, final List defaultPipeline) + DataSet dataSet, final String plugin, final List defaultPipeline) { for (final String defaultPipelinePlugin : defaultPipeline) { @@ -127,31 +126,33 @@ private void runDependencies( } - void runPlugin(String plugin, AtomicReference dataSet) + void runPlugin(String plugin, DataSet dataSet) { - pluginManager.getExtensions(SauronExtension.class, plugin).forEach(pluginExtension -> { + for (SauronExtension pluginExtension : pluginManager.getExtensions(SauronExtension.class, plugin)) + { try { MDC.put("sauron.pluginId", plugin); - MDC.put("sauron.serviceName", dataSet.get().getServiceName()); - MDC.put("sauron.commitId", dataSet.get().getCommitId()); - MDC.put("sauron.buildId", dataSet.get().getBuildId()); - log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get()); - - DataSet newDataSet = getTimerBuilder("sauron.plugin.execution.time") + MDC.put("sauron.serviceName", dataSet.getServiceName()); + MDC.put("sauron.commitId", dataSet.getCommitId()); + MDC.put("sauron.buildId", dataSet.getBuildId()); + log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), + dataSet); + + final DataSet dataSetForLambda = dataSet; + dataSet = getTimerBuilder("sauron.plugin.execution.time") .tag("plugin", plugin) - .tag("service", dataSet.get().getServiceName()) - .tag("commit", dataSet.get().getCommitId()) - .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSet.get())); - dataSet.set(newDataSet); + .tag("service", dataSet.getServiceName()) + .tag("commit", dataSet.getCommitId()) + .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSetForLambda)); meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "success").increment(); - log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get()); + log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet); } catch (final Exception ex) { meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "failure").increment(); - log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, dataSet.get().getServiceName(), dataSet.get().getCommitId(), dataSet.get(), ex); + log.error("Error in plugin '{}' for serviceName={}, commitId={}. DataSet at time of failure: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet, ex); } finally { @@ -160,7 +161,7 @@ void runPlugin(String plugin, AtomicReference dataSet) MDC.remove("sauron.commitId"); MDC.remove("sauron.buildId"); } - }); + } } From 6c8ea6dce6205281b902a899ed323d98d482a71f Mon Sep 17 00:00:00 2001 From: "omid.tavakoli" Date: Thu, 30 Oct 2025 11:13:11 +0100 Subject: [PATCH 7/7] removed spare variable --- .../java/com/freenow/sauron/service/PipelineService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java index 29dd1f67..9febb90b 100644 --- a/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java +++ b/sauron-service/src/main/java/com/freenow/sauron/service/PipelineService.java @@ -139,12 +139,11 @@ void runPlugin(String plugin, DataSet dataSet) log.debug("Applying pluginId: {}. Processing service {} - {}. DataSet BEFORE plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet); - final DataSet dataSetForLambda = dataSet; - dataSet = getTimerBuilder("sauron.plugin.execution.time") + getTimerBuilder("sauron.plugin.execution.time") .tag("plugin", plugin) .tag("service", dataSet.getServiceName()) .tag("commit", dataSet.getCommitId()) - .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSetForLambda)); + .register(meterRegistry).record(() -> pluginExtension.apply(pluginsProperties, dataSet)); meterRegistry.counter("sauron.plugin.executions.total", "plugin", plugin, "result", "success").increment(); log.debug("PluginId: {} applied. Processing service {} - {}. DataSet AFTER plugin execution: {}", plugin, dataSet.getServiceName(), dataSet.getCommitId(), dataSet);