From e2435f38f0e87191be93659415986edd9843e567 Mon Sep 17 00:00:00 2001 From: Michal Szynkiewicz Date: Fri, 5 Jun 2020 18:06:35 +0200 Subject: [PATCH 1/5] some pork --- .../jboss/pnc/bacon/config/ConfigProfile.java | 1 + .../org/jboss/pnc/bacon/config/PigConfig.java | 13 + .../org/jboss/pnc/bacon/pig/PigFacade.java | 29 +- .../jboss/pnc/bacon/pig/impl/PigContext.java | 25 +- .../pnc/bacon/pig/impl/PigProperties.java | 37 + .../pnc/bacon/pig/impl/addons/AddOn.java | 18 +- .../bacon/pig/impl/addons/AddOnFactory.java | 23 +- .../addons/ExtraDeliverableDownloader.java | 17 +- .../BuildLogWithDependencyTrees.java | 88 ++ ...roProfileSmallRyeCommunityDepAnalyzer.java | 63 + .../quarkus/QuarkusCommunityDepAnalyzer.java | 105 +- .../addons/runtime/CommunityDepAnalyzer.java | 50 +- .../addons/runtime/RepoBuildLogProcessor.java | 67 - .../runtime/RuntimeDependenciesAnalyzer.java | 18 +- .../addons/runtime/ThorntailDepAnalyzer.java | 42 + .../impl/addons/spring/BomVerifierAddon.java | 18 +- .../NotYetAlignedFromDependencyTree.java | 6 +- .../pig/impl/common/DeliverableManager.java | 8 +- .../bacon/pig/impl/config/BuildConfig.java | 45 +- .../{Config.java => PigConfiguration.java} | 20 +- .../pig/impl/config/RepoGenerationData.java | 13 +- .../impl/config/RepoGenerationStrategy.java | 2 +- .../bacon/pig/impl/documents/DataRoot.java | 5 +- .../pig/impl/documents/DocumentGenerator.java | 21 +- .../documents/sharedcontent/BrewSearcher.java | 22 +- .../SharedContentReportGenerator.java | 3 +- .../sharedcontent/da/CommunityDependency.java | 30 +- .../sharedcontent/da/CsvExportable.java | 37 + .../da/ThorntailCommunityDependency.java | 35 + .../pig/impl/javadoc/JavadocManager.java | 68 +- .../pig/impl/license/LicenseGenerator.java | 59 +- .../pig/impl/license/LicenseManager.java | 38 +- .../pig/impl/pnc/BuildInfoCollector.java | 8 +- .../pig/impl/pnc/PncEntitiesImporter.java | 45 +- .../bacon/pig/impl/repo/ArtifactVersion.java | 61 + .../impl/repo/ExternalArtifactDownloader.java | 45 +- .../pig/impl/repo/ParentPomDownloader.java | 128 +- .../pnc/bacon/pig/impl/repo/RepoBuilder.java | 141 ++- .../bacon/pig/impl/repo/RepoDescriptor.java | 2 +- .../pnc/bacon/pig/impl/repo/RepoManager.java | 201 ++- .../bacon/pig/impl/repo/RepositoryUtils.java | 83 ++ .../pig/impl/script/ScriptGenerator.java | 48 +- .../sources/SourcesGenerationStrategy.java | 2 +- .../pig/impl/sources/SourcesGenerator.java | 11 + .../bacon/pig/impl/utils/CollectionUtils.java | 6 +- .../pig/impl/utils/FileDownloadUtils.java | 18 +- .../pnc/bacon/pig/impl/utils/FileUtils.java | 87 +- .../jboss/pnc/bacon/pig/impl/utils/GAV.java | 30 +- .../pnc/bacon/pig/impl/utils/JarUtils.java | 22 - .../pnc/bacon/pig/impl/utils/LoggerUtils.java | 51 - .../pig/impl/utils/MavenRepositoryUtils.java | 2 +- .../pnc/bacon/pig/impl/utils/OSCheck.java | 58 - .../pig/impl/utils/OSCommandExecutor.java | 12 +- .../bacon/pig/impl/utils/ResourceUtils.java | 24 +- .../pnc/bacon/pig/impl/utils/XmlUtils.java | 8 +- .../pnc/bacon/pig/impl/utils/indy/Indy.java | 37 + .../{javadoc => utils/pom}/Dependency.java | 4 +- .../pom}/DependencyManagement.java | 4 +- .../impl/{javadoc => utils/pom}/Profile.java | 4 +- .../impl/{javadoc => utils/pom}/Profiles.java | 4 +- .../impl/{javadoc => utils/pom}/Project.java | 4 +- pig/src/main/resources/ARTIFACT_LIST.ftl | 1 + .../resources/DUPLICATE_ARTIFACT_LIST.ftl | 1 + pig/src/main/resources/README.ftl | 56 + .../main/resources/REPOSITORY_CONTENTS.ftl | 1 + .../main/resources/REPOSITORY_COORDINATES.ftl | 4 + pig/src/main/resources/generate-nvr-list.sh | 47 + pig/src/main/resources/indy-settings.xml | 56 + pig/src/main/resources/indy-temp-settings.xml | 69 + .../resources/koji-build-finder/config.json | 10 + .../resources/license-generator.properties | 4 + pig/src/main/resources/pom-template.xml | 63 + pig/src/main/resources/release.sh | 38 + pig/src/main/resources/repository-README.md | 4 + .../resources/repository-example-settings.xml | 54 + .../main/resources/rh-license-exceptions.json | 1113 +++++++++++++++++ pig/src/main/resources/settings-template.xml | 57 + 77 files changed, 3090 insertions(+), 664 deletions(-) create mode 100644 config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/{Config.java => PigConfiguration.java} (92%) create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/JarUtils.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/LoggerUtils.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/OSCheck.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/indy/Indy.java rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Dependency.java (96%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/DependencyManagement.java (91%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Profile.java (94%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Profiles.java (89%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Project.java (95%) create mode 100644 pig/src/main/resources/ARTIFACT_LIST.ftl create mode 100644 pig/src/main/resources/DUPLICATE_ARTIFACT_LIST.ftl create mode 100644 pig/src/main/resources/README.ftl create mode 100644 pig/src/main/resources/REPOSITORY_CONTENTS.ftl create mode 100644 pig/src/main/resources/REPOSITORY_COORDINATES.ftl create mode 100755 pig/src/main/resources/generate-nvr-list.sh create mode 100644 pig/src/main/resources/indy-settings.xml create mode 100644 pig/src/main/resources/indy-temp-settings.xml create mode 100644 pig/src/main/resources/koji-build-finder/config.json create mode 100644 pig/src/main/resources/license-generator.properties create mode 100644 pig/src/main/resources/pom-template.xml create mode 100644 pig/src/main/resources/release.sh create mode 100644 pig/src/main/resources/repository-README.md create mode 100644 pig/src/main/resources/repository-example-settings.xml create mode 100644 pig/src/main/resources/rh-license-exceptions.json create mode 100644 pig/src/main/resources/settings-template.xml diff --git a/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java b/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java index ad3c096bc..544bfc250 100644 --- a/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java +++ b/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java @@ -29,6 +29,7 @@ public class ConfigProfile { private PncConfig pnc; private DaConfig da; private IndyConfig indy; + private PigConfig pig; private KeycloakConfig keycloak; private Map> addOns; diff --git a/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java b/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java new file mode 100644 index 000000000..410512c31 --- /dev/null +++ b/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java @@ -0,0 +1,13 @@ +package org.jboss.pnc.bacon.config; + +import lombok.Data; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +@Data +public class PigConfig { + private String kojiHubUrl; + private String licenseServiceUrl; + private String indyUrl; +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java index 104b055a5..68ac628a4 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java @@ -20,7 +20,7 @@ import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; import org.jboss.pnc.bacon.pig.impl.addons.AddOnFactory; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.DocumentGenerator; import org.jboss.pnc.bacon.pig.impl.javadoc.JavadocManager; import org.jboss.pnc.bacon.pig.impl.license.LicenseManager; @@ -190,7 +190,9 @@ public static String run( } public static void generateScripts() { - ScriptGenerator scriptGenerator = new ScriptGenerator(context().getConfig(), context().getDeliverables()); + ScriptGenerator scriptGenerator = new ScriptGenerator( + context().getPigConfiguration(), + context().getDeliverables()); scriptGenerator.generateReleaseScripts( context().getPncImportResult().getMilestone(), context().getRepositoryData().getRepositoryPath(), @@ -202,7 +204,7 @@ public static void generateScripts() { public static void generateDocuments() { DocumentGenerator docGenerator = new DocumentGenerator( - context().getConfig(), + context().getPigConfiguration(), context().getReleasePath(), context().getExtrasPath(), context().getDeliverables()); @@ -212,7 +214,7 @@ public static void generateDocuments() { public static void prepareSharedContentAnalysis() { try { DocumentGenerator docGenerator = new DocumentGenerator( - context().getConfig(), + context().getPigConfiguration(), context().getReleasePath(), context().getExtrasPath(), context().getDeliverables()); @@ -223,12 +225,12 @@ public static void prepareSharedContentAnalysis() { } public static void generateSources() { - Config config = context().getConfig(); + PigConfiguration pigConfiguration = context().getPigConfiguration(); Map builds = context().getBuilds(); RepositoryData repo = context().getRepositoryData(); SourcesGenerator sourcesGenerator = new SourcesGenerator( - config.getFlow().getSourcesGeneration(), - config.getTopLevelDirectoryPrefix() + "src", + pigConfiguration.getFlow().getSourcesGeneration(), + pigConfiguration.getTopLevelDirectoryPrefix() + "src", context().getReleasePath() + context().getDeliverables().getSourceZipName()); sourcesGenerator.generateSources(builds, repo); } @@ -275,7 +277,7 @@ private static RepositoryData parseRepository(File repositoryZipPath) { public static void triggerAddOns() { AddOnFactory .listAddOns( - context().getConfig(), + context().getPigConfiguration(), context().getBuilds(), context().getReleasePath(), context().getExtrasPath(), @@ -288,7 +290,7 @@ public static void triggerAddOns() { public static RepositoryData generateRepo(boolean removeGeneratedM2Dups) { PigContext context = context(); RepoManager repoManager = new RepoManager( - context.getConfig(), + context.getPigConfiguration(), context().getReleasePath(), context().getDeliverables(), context().getBuilds(), @@ -308,15 +310,16 @@ private static Map getBuilds(ImportResult importResult) { public static void generateLicenses() { PigContext context = context(); - Config config = context.getConfig(); + PigConfiguration pigConfiguration = context.getPigConfiguration(); RepositoryData repo = context.getRepositoryData(); Map builds = context.getBuilds(); - new LicenseManager(config, context.getReleasePath(), context.getDeliverables(), builds, repo).prepare(); + new LicenseManager(pigConfiguration, context.getReleasePath(), context.getDeliverables(), builds, repo) + .prepare(); } public static void generateJavadoc() { - Config config = context().getConfig(); + PigConfiguration pigConfiguration = context().getPigConfiguration(); Map builds = context().getBuilds(); - new JavadocManager(config, context().getReleasePath(), context().getDeliverables(), builds).prepare(); + new JavadocManager(pigConfiguration, context().getReleasePath(), context().getDeliverables(), builds).prepare(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java index 69c23399a..6425ec43d 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.ImportResult; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -50,7 +50,7 @@ public class PigContext { "pig.context.dir", getProperty("java.io.tmpdir") + File.separator + "pig-context"); - private Config config; // TODO merge config instead of setting it? + private PigConfiguration pigConfiguration; // TODO merge config instead of setting it? private ImportResult pncImportResult; private Map builds; private RepositoryData repositoryData; @@ -60,14 +60,14 @@ public class PigContext { private String releasePath; private String extrasPath; - public void setConfig(Config config) { - this.config = config; + public void setPigConfiguration(PigConfiguration pigConfiguration) { + this.pigConfiguration = pigConfiguration; if (deliverables == null) { String prefix = String.format( "%s-%s.%s", - config.getOutputPrefixes().getReleaseFile(), - config.getVersion(), - config.getMilestone()); + pigConfiguration.getOutputPrefixes().getReleaseFile(), + pigConfiguration.getVersion(), + pigConfiguration.getMilestone()); deliverables = new Deliverables(); @@ -77,13 +77,14 @@ public void setConfig(Config config) { deliverables.setJavadocZipName(prefix + "-javadoc.zip"); deliverables.setNvrListName(prefix + "-nvr-list.txt"); } - configureTargetDirectories(config); + configureTargetDirectories(pigConfiguration); } - private void configureTargetDirectories(Config config) { - String productPrefix = config.getProduct().prefix(); + private void configureTargetDirectories(PigConfiguration pigConfiguration) { + String productPrefix = pigConfiguration.getProduct().prefix(); targetPath = "target"; // TODO: a way to customize it - releasePath = targetPath + "/" + productPrefix + "-" + config.getVersion() + "." + config.getMilestone() + "/"; + releasePath = targetPath + "/" + productPrefix + "-" + pigConfiguration.getVersion() + "." + + pigConfiguration.getMilestone() + "/"; File releaseDirectory = Paths.get(releasePath).toFile(); if (!releaseDirectory.isDirectory()) { @@ -112,7 +113,7 @@ public void loadConfig(String config) { File configFile = new File(config); if (configFile.exists()) { try (FileInputStream configStream = new FileInputStream(configFile)) { - setConfig(Config.load(configStream)); + setPigConfiguration(PigConfiguration.load(configStream)); } catch (IOException e) { throw new RuntimeException("Failed to read config file: " + configFile.getAbsolutePath(), e); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java new file mode 100644 index 000000000..245a6f968 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java @@ -0,0 +1,37 @@ +package org.jboss.pnc.bacon.pig.impl; + +import lombok.Getter; + +import java.util.Properties; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +@Getter +public class PigProperties { + + private boolean temporary; + private boolean removeGeneratedM2Dups; + private boolean skipBranchCheck; + + private PigProperties() { + } + + private static PigProperties instance; + + public static synchronized void init(Properties properties) { + instance = new PigProperties(); + instance.removeGeneratedM2Dups = Boolean.TRUE.toString() + .equalsIgnoreCase(properties.getProperty("removeGeneratedM2Dups", "false")); + instance.skipBranchCheck = Boolean.TRUE.toString() + .equalsIgnoreCase(properties.getProperty("skipBranchCheck", "false")); + instance.temporary = Boolean.TRUE.toString().equalsIgnoreCase(properties.getProperty("temporary", "false")); + } + + public static synchronized PigProperties get() { + if (instance == null) { + init(System.getProperties()); + } + return instance; + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java index d484665e1..281e2bdd2 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java @@ -17,7 +17,7 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import java.util.Map; @@ -27,24 +27,28 @@ * Date: 12/11/17 */ public abstract class AddOn { - protected final Config config; + protected final PigConfiguration pigConfiguration; protected final Map builds; protected final String releasePath; protected final String extrasPath; - protected AddOn(Config config, Map builds, String releasePath, String extrasPath) { - this.config = config; + protected AddOn( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + this.pigConfiguration = pigConfiguration; this.builds = builds; this.releasePath = releasePath; this.extrasPath = extrasPath; } public boolean shouldRun() { - return config.getAddons().keySet().contains(getName()); + return pigConfiguration.getAddons().containsKey(getName()); } - public Map getConfig() { - return config.getAddons().get(getName()); + public Map getPigConfiguration() { + return pigConfiguration.getAddons().get(getName()); } protected abstract String getName(); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java index d31713394..72659088f 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java @@ -17,11 +17,12 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; +import org.jboss.pnc.bacon.pig.impl.addons.microprofile.MicroProfileSmallRyeCommunityDepAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.quarkus.QuarkusCommunityDepAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.runtime.RuntimeDependenciesAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.spring.BomVerifierAddon; import org.jboss.pnc.bacon.pig.impl.addons.vertx.NotYetAlignedFromDependencyTree; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -35,23 +36,25 @@ */ public class AddOnFactory { - private AddOnFactory() { - } - public static List listAddOns( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath, Deliverables deliverables) { ArrayList resultList = new ArrayList<>(); - resultList.add(new RuntimeDependenciesAnalyzer(config, builds, releasePath, extrasPath)); - resultList.add(new ExtraDeliverableDownloader(config, builds, releasePath, extrasPath)); - resultList.add(new BomVerifierAddon(config, builds, releasePath, extrasPath)); - resultList.add(new NotYetAlignedFromDependencyTree(config, builds, releasePath, extrasPath)); - resultList.add(new QuarkusCommunityDepAnalyzer(config, builds, releasePath, extrasPath, deliverables)); + resultList.add(new RuntimeDependenciesAnalyzer(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new ExtraDeliverableDownloader(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new BomVerifierAddon(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new NotYetAlignedFromDependencyTree(pigConfiguration, builds, releasePath, extrasPath)); + resultList + .add(new QuarkusCommunityDepAnalyzer(pigConfiguration, builds, releasePath, extrasPath, deliverables)); + resultList.add(new MicroProfileSmallRyeCommunityDepAnalyzer(pigConfiguration, builds, releasePath, extrasPath)); return resultList; } + + private AddOnFactory() { + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java index 63fa0efa6..07f105f73 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java @@ -17,7 +17,7 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -33,11 +33,11 @@ public class ExtraDeliverableDownloader extends AddOn { protected ExtraDeliverableDownloader( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -45,10 +45,11 @@ protected String getName() { return "extraDeliverablesDownloader"; } + @SuppressWarnings("unchecked") @Override public void trigger() { - // noinspection unchecked - getConfig().forEach((key, artifacts) -> downloadArtifacts(key, (List>) artifacts)); + getPigConfiguration() + .forEach((key, artifacts) -> downloadArtifacts(key, (List>) artifacts)); } private void downloadArtifacts(String buildName, List> artifacts) { @@ -66,9 +67,9 @@ private void downloadArtifact(PncBuild build, String pattern, String suffix) { private String constructFileName(String suffix) { return String.format( "%s-%s.%s-%s", - config.getOutputPrefixes().getReleaseFile(), - config.getVersion(), - config.getMilestone(), + pigConfiguration.getOutputPrefixes().getReleaseFile(), + pigConfiguration.getVersion(), + pigConfiguration.getMilestone(), suffix); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java new file mode 100644 index 000000000..b7d1a2e22 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java @@ -0,0 +1,88 @@ +package org.jboss.pnc.bacon.pig.impl.addons.microprofile; + +import com.google.common.collect.Sets; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class BuildLogWithDependencyTrees { + private static final Logger log = LoggerFactory.getLogger(BuildLogWithDependencyTrees.class); + + private static final Pattern dependencyTreeStart = Pattern + .compile("^\\[INFO] --- maven-dependency-plugin:.*?:tree .*? @ (.*?) ---$"); + private static final Pattern dependencyTreeEnd = Pattern.compile("^\\[INFO][\\s-]*$"); + private static final Set relevantScopes = Sets.newHashSet("compile", "runtime"); + + public final Map> communityGavsForModules; + + public BuildLogWithDependencyTrees(List buildLogLines) { + communityGavsForModules = parseDependencyTreeInvocations(buildLogLines); + } + + private static Map> parseDependencyTreeInvocations(List buildLogLines) { + Map> result = new HashMap<>(); + + String currentModuleName = null; + List currentModuleDependencyTree = null; + + for (String line : buildLogLines) { + Matcher matcher = dependencyTreeStart.matcher(line); + if (matcher.matches()) { + currentModuleName = matcher.group(1); + currentModuleDependencyTree = new ArrayList<>(); + continue; + } + + if (dependencyTreeEnd.matcher(line).matches() && currentModuleDependencyTree != null) { + result.put(currentModuleName, communityGavsInDepTree(currentModuleDependencyTree)); + currentModuleName = null; + currentModuleDependencyTree = null; + continue; + } + + if (currentModuleDependencyTree != null) { + currentModuleDependencyTree.add(line); + } + } + + return result; + } + + private static Set communityGavsInDepTree(List depTreeOutput) { + return depTreeOutput.stream() + .filter(l -> l.startsWith("[INFO] ")) + .map(BuildLogWithDependencyTrees::parseLineToGav) + .filter(Objects::nonNull) + .filter(GAV::isCommunity) + .collect(Collectors.toSet()); + } + + private static GAV parseLineToGav(String mvnDepTreeLine) { + String gavString = mvnDepTreeLine.replaceFirst("\\[INFO] [+|\\\\\\-\\s]+", ""); + String[] splitGav = gavString.split(":"); + if (splitGav.length < 5 || !relevantScopes.contains(splitGav[splitGav.length - 1])) { + return null; + } + switch (splitGav.length) { + case 5: + return new GAV(splitGav[0], splitGav[1], splitGav[3], splitGav[2]); + case 6: + return new GAV(splitGav[0], splitGav[1], splitGav[4], splitGav[2], splitGav[3]); + default: + log.warn( + "suspicious line in the dependency tree '{}', assuming it's not a dependency and skipping", + gavString); + return null; + } + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java new file mode 100644 index 000000000..607094f3c --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java @@ -0,0 +1,63 @@ +package org.jboss.pnc.bacon.pig.impl.addons.microprofile; + +import org.jboss.pnc.bacon.pig.impl.addons.AddOn; +import org.jboss.pnc.bacon.pig.impl.addons.runtime.CommunityDepAnalyzer; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Performs community dependency analysis on all builds (in "our" build group, that is). Assumes that each build + * includes {@code mvn dependency:tree} in its build script. Community dependencies from modules whose names contain + * {@code test} or {@code tck} are ignored. Only dependencies in the {@code compile} and {@code runtime} scopes are + * included. + */ +public class MicroProfileSmallRyeCommunityDepAnalyzer extends AddOn { + private static final Logger log = LoggerFactory.getLogger(MicroProfileSmallRyeCommunityDepAnalyzer.class); + + public static final String NAME = "microProfileSmallRyeCommunityDepAnalyzer"; + + public MicroProfileSmallRyeCommunityDepAnalyzer( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + super(pigConfiguration, builds, releasePath, extrasPath); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + public void trigger() { + log.info("Running MicroProfileSmallRyeCommunityDepAnalyzer"); + + Set allGavs = new HashSet<>(); + for (PncBuild build : builds.values()) { + Set gavs = new BuildLogWithDependencyTrees(build.getBuildLog()).communityGavsForModules.entrySet() + .stream() + .filter(e -> !e.getKey().contains("test") && !e.getKey().contains("tck")) + .flatMap(e -> e.getValue().stream()) + .collect(Collectors.toSet()); + + Path targetPath = Paths.get(extrasPath, "community-dependencies-" + build.getName() + ".csv"); + new CommunityDepAnalyzer(gavs).generateAnalysis(targetPath.toAbsolutePath().toString()); + + allGavs.addAll(gavs); + } + + Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); + new CommunityDepAnalyzer(allGavs).generateAnalysis(targetPath.toAbsolutePath().toString()); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java index 2e3d87b1c..d85f4b8ee 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java @@ -5,17 +5,24 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.collect.Sets; import lombok.SneakyThrows; - +import org.apache.maven.model.Dependency; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; import org.jboss.pnc.bacon.pig.impl.addons.runtime.CommunityDepAnalyzer; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; -import org.jboss.pnc.bacon.pig.impl.utils.*; - +import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.jboss.pnc.bacon.pig.impl.utils.MavenRepositoryUtils; +import org.jboss.pnc.bacon.pig.impl.utils.OSCommandExecutor; +import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -30,6 +37,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -61,12 +69,12 @@ public class QuarkusCommunityDepAnalyzer extends AddOn { } public QuarkusCommunityDepAnalyzer( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath, Deliverables deliverables) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); this.deliverables = deliverables; } @@ -83,7 +91,7 @@ public void trigger() { Path repoZipPath = Paths.get(releasePath + deliverables.getRepositoryZipName()); unpackRepository(repoZipPath); - String additionalRepository = (String) getConfig().get("additionalRepository"); + String additionalRepository = (String) getPigConfiguration().get("additionalRepository"); String settingsSelector = ""; if (additionalRepository != null) { try { @@ -114,6 +122,15 @@ public void trigger() { Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); depAnalyzer.generateAnalysis(targetPath.toAbsolutePath().toString()); + + Set problematicDeps = gatherProblematicDeps(); + Path problematicDepsOut = Paths.get(extrasPath, "nonexistent-redhat-deps.txt"); + + try (BufferedWriter writer = Files.newBufferedWriter(problematicDepsOut)) { + writer.write(String.join("\n", problematicDeps)); + } catch (IOException e) { + throw new RuntimeException("Failed to write problematic dependencies to the output file", e); + } } private Path buildReactiveProject(String settingsSelector) { @@ -130,7 +147,7 @@ private Path buildNonReactiveProject(String settingsSelector) { private Collection skippedExtensions() { // noinspection unchecked - Collection skipped = (Collection) getConfig().get("skippedExtensions"); + Collection skipped = (Collection) getPigConfiguration().get("skippedExtensions"); return skipped == null ? Collections.emptyList() : skipped; } @@ -156,8 +173,8 @@ private void unpackRepository(Path repoZipPath) { @SneakyThrows private Set listDependencies(Path projectPath, Path depThreeOut, String settingsSelector) { - List result = OSCommandExecutor - .runCommandIn("mvn dependency:tree" + repoDefinition + settingsSelector, projectPath); + List result = OSCommandExecutor // mstodo!!! + .runCommandIn("mvn dependency:tree " + repoDefinition + settingsSelector, projectPath); Files.write(depThreeOut, result); return depTreeToNonRedhatGAVs(result); @@ -279,4 +296,72 @@ private List findAllByExtension(String extension) { } return visitor.getFilePaths(); } + + private Set gatherProblematicDeps() { + Set problemmaticDeps = new TreeSet<>(); + problemmaticDeps.addAll(checkBomContents(".*/io/quarkus/quarkus-bom/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/io/quarkus/quarkus-bom-deployment/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/com/redhat/quarkus/quarkus-product-bom/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/com/redhat/quarkus/quarkus-product-bom-deployment/.*\\.pom")); + return problemmaticDeps; + } + + private Collection checkBomContents(String bomLocator) { + String quarkusRuntimeBom = repoZipContents.stream() + // todo file separator for Wi***ws may be different + .filter(file -> file.matches(bomLocator)) + .findAny() + .get(); + return checkReferencesInRepo(quarkusRuntimeBom); + } + + // mstodo store the result properly + // mstodo add it add on trigger + private Collection checkReferencesInRepo(String quarkusRuntimeBom) { + try { + String str = "/maven-repository/"; + int repoDirIdx = quarkusRuntimeBom.indexOf(str); + quarkusRuntimeBom = quarkusRuntimeBom.substring(repoDirIdx + str.length()); + Path bomFile = repoPath.resolve(quarkusRuntimeBom).toAbsolutePath(); + Model model = new MavenXpp3Reader().read(Files.newInputStream(bomFile)); + + List dependencies = model.getDependencyManagement().getDependencies(); + + return dependencies.stream() + .filter(dep -> isRHAndMissing(dep, model)) + .map( + dep -> String.format( + "'%s:%s:%s',", + dep.getGroupId(), + dep.getArtifactId(), + deVar(model, dep.getVersion()))) + .collect(Collectors.toSet()); + } catch (XmlPullParserException | IOException e) { + log.error("Parsing error when generating quarkus artifact references", e); + return Collections.emptySet(); + } + } + + private boolean isRHAndMissing(Dependency dependency, Model model) { + String version = dependency.getVersion(); + version = deVar(model, version); + if (!version.contains("redhat")) { + return false; + } + GAV gav = new GAV( + dependency.getGroupId(), + dependency.getArtifactId(), + version, + dependency.getType() != null && dependency.getType().equals("jar") ? dependency.getType() : "jar"); + Path filePath = repoPath.resolve(gav.toVersionPath()).resolve(gav.toFileName()); + return !filePath.toFile().exists(); + } + + private String deVar(Model model, String version) { + if (version.startsWith("$")) { + version = version.substring(2, version.length() - 1); + version = model.getProperties().getProperty(version); + } + return version; + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java index 10ad48c4a..137f8f406 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java @@ -18,6 +18,7 @@ package org.jboss.pnc.bacon.pig.impl.addons.runtime; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CommunityDependency; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CsvExportable; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.DADao; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.slf4j.Logger; @@ -27,6 +28,7 @@ import java.io.FileWriter; import java.util.Collection; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -39,40 +41,34 @@ public class CommunityDepAnalyzer { private final DADao daDao; private final List dependencies; - private final List downloadedForSwarm; + private final Function, List> enricher; + private boolean skipDa = false; - public CommunityDepAnalyzer(List dependencyLines, List swarmLog) { + public CommunityDepAnalyzer( + List dependencyLines, + Function, List> enricher) { daDao = DADao.getInstance(); - downloadedForSwarm = swarmLog.stream() - .filter(line -> line.startsWith("Downloaded")) - .filter(line -> line.contains(".jar")) - // lines are of the form: Downloaded: http://... (some add. info) - .map(l -> l.split("\\s+")[1]) - .sorted() - .collect(Collectors.toList()); dependencies = dependencyLines.stream().map(CommunityDependency::new).collect(Collectors.toList()); + this.enricher = enricher; } public CommunityDepAnalyzer(Collection gavs) { dependencies = gavs.stream().map(CommunityDependency::new).collect(Collectors.toList()); daDao = null; - downloadedForSwarm = null; + enricher = null; } public File generateAnalysis(String path) { try { - if (!skipDa) { - analyzeDAResults(); - } - analyzeSwarmBuildLog(); log.info("generating analysis to {} ...", path); + List csvContents = analyze(); File csvFile = new File(path); try (FileWriter writer = new FileWriter(csvFile)) { writer.append( - "Community dependencies in Swarm;;Productized counterpart;Other productized versions;Used in the Swarm build\n"); - dependencies.forEach(d -> d.appendToCsv(writer)); + "Community dependencies;;Productized counterpart;Other productized versions;Additional info\n"); + csvContents.forEach(d -> d.appendToCsv(writer)); } log.info("DONE"); @@ -82,22 +78,20 @@ public File generateAnalysis(String path) { } } - protected List analyzeDAResults() { - dependencies.parallelStream().forEach(daDao::fillDaData); - return dependencies; + protected List analyze() { + if (!skipDa) { + analyzeDAResults(); + } + return productSpecificAnalysis(); } - protected List analyzeSwarmBuildLog() { - dependencies.parallelStream().forEach(this::addSwarmBuildDependencies); - return dependencies; + private List productSpecificAnalysis() { + return enricher != null ? enricher.apply(dependencies) : dependencies; } - private void addSwarmBuildDependencies(CommunityDependency communityDependency) { - List swarmBuildDownloads = downloadedForSwarm.stream() - .filter(d -> d.contains(communityDependency.toPathSubstring())) - .map(l -> l.substring(l.lastIndexOf("/") + 1)) - .collect(Collectors.toList()); - communityDependency.setUsedForSwarm(swarmBuildDownloads); + protected List analyzeDAResults() { + dependencies.parallelStream().forEach(daDao::fillDaData); + return dependencies; } public void skipDa(boolean skip) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java deleted file mode 100644 index f8335265b..000000000 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2017 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.bacon.pig.impl.addons.runtime; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import static org.apache.commons.lang3.StringUtils.join; - -/** - * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
- * Date: 6/5/17 - */ -public class RepoBuildLogProcessor { - private static final String START_TAG = "swarm_repository_listing_begin"; - private static final String END_TAG = "swarm_repository_listing_end"; - - public static List getList(String header, List buildLog) { - Iterator logIterator = buildLog.iterator(); - while (logIterator.hasNext()) { - if (header.equals(logIterator.next())) { - break; - } - } - if (logIterator.hasNext()) { - if (START_TAG.equals(logIterator.next()) && logIterator.hasNext()) { - return takeUntilTheEnd(logIterator); - } else { - throw new IllegalArgumentException( - "Malformed repo log listing for header: " + header + " in the log: " + join(buildLog, "\n")); - } - } else { - throw new IllegalArgumentException("Unable to find " + header + " in the log: " + join(buildLog, "\n")); - } - } - - private static List takeUntilTheEnd(Iterator logIterator) { - List resultList = new ArrayList<>(); - while (logIterator.hasNext()) { - String nextLine = logIterator.next(); - if (END_TAG.equals(nextLine)) { - break; - } - resultList.add(nextLine); - } - return resultList; - } - - private RepoBuildLogProcessor() { - } -} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java index 28384e8cc..4ae787c59 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java @@ -19,7 +19,7 @@ import org.apache.commons.io.IOUtils; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.slf4j.Logger; @@ -42,11 +42,11 @@ public class RuntimeDependenciesAnalyzer extends AddOn { private static final Logger log = LoggerFactory.getLogger(RuntimeDependenciesAnalyzer.class); public RuntimeDependenciesAnalyzer( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -59,9 +59,9 @@ public void trigger() { log.info("Running RuntimeDependenciesAnalyzer"); File workDir = FileUtils.mkTempDir("runtimeDepAnalyzer"); - String buildName = (String) getConfig().get("downloadFrom"); - String regex = (String) getConfig().get("matching"); - String referenceBuildName = (String) getConfig().get("referenceBuild"); + String buildName = (String) getPigConfiguration().get("downloadFrom"); + String regex = (String) getPigConfiguration().get("matching"); + String referenceBuildName = (String) getPigConfiguration().get("referenceBuild"); File dependencyListPath = new File(workDir, "runtime-dependency-list.txt"); builds.get(buildName).downloadArtifact(regex, dependencyListPath); @@ -82,8 +82,10 @@ public void trigger() { Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); - CommunityDepAnalyzer analyzer = new CommunityDepAnalyzer(communityDependencies, buildLog); - analyzer.skipDa(false); // TODO: + CommunityDepAnalyzer analyzer = new CommunityDepAnalyzer( + communityDependencies, + new ThorntailDepAnalyzer(buildLog)); + analyzer.skipDa(false); analyzer.generateAnalysis(targetPath.toAbsolutePath().toString()); log.info("Done"); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java new file mode 100644 index 000000000..2793d2076 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java @@ -0,0 +1,42 @@ +package org.jboss.pnc.bacon.pig.impl.addons.runtime; + +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CommunityDependency; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CsvExportable; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.ThorntailCommunityDependency; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +public class ThorntailDepAnalyzer implements Function, List> { + private final List downloadedForSwarm; + + public ThorntailDepAnalyzer(List swarmLog) { + downloadedForSwarm = swarmLog.stream() + .filter(line -> line.startsWith("Downloaded")) + .filter(line -> line.contains(".jar")) + // lines are of the form: Downloaded: http://... (some add. info) + .map(l -> l.split("\\s+")[1]) + .sorted() + .collect(Collectors.toList()); + } + + @Override + public List apply(List communityDependencies) { + return communityDependencies.parallelStream() + .map(this::toThorntailCommunityDependency) + .collect(Collectors.toList()); + } + + private ThorntailCommunityDependency toThorntailCommunityDependency(CommunityDependency communityDependency) { + List swarmBuildDownloads = downloadedForSwarm.stream() + .filter(d -> d.contains(communityDependency.toPathSubstring())) + .map(l -> l.substring(l.lastIndexOf("/") + 1)) + .collect(Collectors.toList()); + return new ThorntailCommunityDependency(communityDependency, swarmBuildDownloads); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java index b4f527e50..cf5624aed 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java @@ -17,7 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.MRRCSearcher; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.GAV; @@ -47,8 +47,12 @@ public class BomVerifierAddon extends AddOn { private final MRRCSearcher mrrcSearcher = MRRCSearcher.getInstance(); private List unreleasedWhitelist; - public BomVerifierAddon(Config config, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + public BomVerifierAddon( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -75,9 +79,10 @@ private List readUnreleasedWhitelist() { }).collect(Collectors.toList()); } + @SuppressWarnings("unchecked") private List getListFromConfig() { try { - Map config = getConfig(); + Map config = getPigConfiguration(); if (config != null) { List result = (List) config.get("allowUnreleased"); return result == null ? emptyList() : result; @@ -111,10 +116,11 @@ private boolean unreleased(GAV gav) { } protected Stream getDependencyGavs() { - PncBuild build = builds.get(config.getFlow().getRepositoryGeneration().getSourceBuild()); + PncBuild build = builds.get(pigConfiguration.getFlow().getRepositoryGeneration().getSourceBuild()); File bom = new File("bom"); - build.findArtifactByFileName(config.getFlow().getRepositoryGeneration().getSourceArtifact()).downloadTo(bom); + build.findArtifactByFileName(pigConfiguration.getFlow().getRepositoryGeneration().getSourceArtifact()) + .downloadTo(bom); List nodeList = listNodes(bom, "//dependencies/dependency"); Map properties = XmlUtils.getProperties(bom); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java index 8eda8c30c..9688e197a 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java @@ -16,7 +16,7 @@ package org.jboss.pnc.bacon.pig.impl.addons.vertx; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,11 +39,11 @@ public class NotYetAlignedFromDependencyTree extends AddOn { private static final Logger log = LoggerFactory.getLogger(NotYetAlignedFromDependencyTree.class); public NotYetAlignedFromDependencyTree( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java index cb71369e6..e224b928a 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java @@ -18,8 +18,8 @@ package org.jboss.pnc.bacon.pig.impl.common; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; @@ -56,18 +56,18 @@ public abstract class DeliverableManager, private static final Logger log = LoggerFactory.getLogger(DeliverableManager.class); - protected final Config config; + protected final PigConfiguration pigConfiguration; protected final String releasePath; protected final Deliverables deliverables; protected final Map builds; protected final File workDir; protected DeliverableManager( - Config config, + PigConfiguration pigConfiguration, String releasePath, Deliverables deliverables, Map builds) { - this.config = config; + this.pigConfiguration = pigConfiguration; this.releasePath = releasePath; this.deliverables = deliverables; this.builds = builds; diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java index 359e2417c..99480d4bc 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.pnc.GitRepoInspector; import org.jboss.pnc.dto.BuildConfiguration; import org.jboss.pnc.dto.SCMRepository; @@ -39,6 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -60,13 +62,12 @@ public class BuildConfig { private String name; private String project; private String buildScript; - private String buildType; - private String scmUrl; private String externalScmUrl; private String scmRevision; private String description; private String environmentId; + private List dependencies = new ArrayList<>(); /** * build pod memory in GB @@ -76,11 +77,11 @@ public class BuildConfig { private Set customPmeParameters = new TreeSet<>(); private Set extraRepositories = new TreeSet<>(); + private Boolean branchModified; + private String buildType; private String executionRoot; - private List dependencies = new ArrayList<>(); private Set alignmentParameters = new TreeSet<>(); - private Boolean branchModified; /** * Set the defaults of buildConfig if not explicitly specified @@ -89,6 +90,7 @@ public class BuildConfig { * * @param defaults */ + @SuppressWarnings("rawtypes") public void setDefaults(BuildConfig defaults) { try { for (Field f : BuildConfig.class.getDeclaredFields()) { @@ -121,6 +123,20 @@ public void sanitizebuildScript() { buildScript = buildScript.trim(); } + @JsonIgnore + public synchronized boolean isBranchModified(BuildConfiguration oldVersion) { + if (PigProperties.get().isSkipBranchCheck()) { + return false; + } + if (branchModified == null) { + branchModified = GitRepoInspector.isModifiedBranch( + oldVersion.getId(), + oldVersion.getScmRepository().getInternalUrl(), + getScmRevision()); + } + return branchModified; + } + @JsonIgnore public boolean isTheSameAs(BuildConfiguration old) { return old != null && StringUtils.equals(name, old.getName()) @@ -138,16 +154,6 @@ private Set getAlignmentParameters(BuildConfiguration old) { return Arrays.stream(parametersAsString.split(",")).collect(Collectors.toSet()); } - private synchronized boolean isBranchModified(BuildConfiguration oldVersion) { - if (branchModified == null) { - branchModified = GitRepoInspector.isModifiedBranch( - oldVersion.getId(), - oldVersion.getScmRepository().getInternalUrl(), - getScmRevision()); - } - return branchModified; - } - private boolean urlsEqual(SCMRepository repo) { return StringUtils.equals(externalScmUrl, repo.getExternalUrl()) || StringUtils.equals(scmUrl, repo.getInternalUrl()); @@ -162,13 +168,13 @@ public Map getGenericParameters(BuildConfiguration oldConfig, bo } Map result = new HashMap<>(); - String oldForceValue = oldConfig == null ? "" : oldConfig.getParameters().getOrDefault(BUILD_FORCE, ""); - String forceValue = forceRebuild ? randomAlphabetic(5) : oldForceValue; + Optional oldForceValue = oldConfig == null ? Optional.empty() + : Optional.ofNullable(oldConfig.getParameters().get(BUILD_FORCE)); + Optional forceValue = forceRebuild ? Optional.of(randomAlphabetic(5)) : oldForceValue; + forceValue.ifPresent(val -> result.put(BUILD_FORCE, val)); String dependencyExclusions = String.join(" ", alignmentParameters); - result.put("ALIGNMENT_PARAMETERS", dependencyExclusions); - result.put(BUILD_FORCE, forceValue); if (buildPodMemory != null) { result.put("BUILDER_POD_MEMORY", buildPodMemory.toString()); @@ -225,6 +231,9 @@ public boolean matchesRepository(SCMRepository repository) { } private boolean areSameRepoUrls(String scmUrl1, String scmUrl2) { + if (scmUrl1 == null ^ scmUrl2 == null) { + throw new RuntimeException("trying to compare null and non-null scm url: " + scmUrl1 + ", " + scmUrl2); + } String normalizedUrl1 = normalize(scmUrl1); String normalizedUrl2 = normalize(scmUrl2); return normalizedUrl1.equals(normalizedUrl2); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java similarity index 92% rename from pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java rename to pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java index b041a82c5..da548baca 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java @@ -52,8 +52,8 @@ * Date: 11/28/17 */ @Data -public class Config { - private static final Logger log = LoggerFactory.getLogger(Config.class); +public class PigConfiguration { + private static final Logger log = LoggerFactory.getLogger(PigConfiguration.class); private @NotBlank ProductConfig product; private @NotBlank String version; @@ -155,7 +155,7 @@ public static InputStream preProcess(InputStream buildConfig, String buildVarsOv } Map variables = readVariables(contents, buildVarsOverrides); - Integer passes = 0; + int passes = 0; // We also have to take into account variable used inside variables, just keep going over until // they have all expanded, maxTries will be hit if one is left over after that many List matches; @@ -187,7 +187,7 @@ public static InputStream preProcess(InputStream buildConfig, String buildVarsOv return stream; } - public static Config load(File buildConfigFile, String buildVarsOverrides) { + public static PigConfiguration load(File buildConfigFile, String buildVarsOverrides) { try (InputStream configStream = new FileInputStream(buildConfigFile)) { return load(configStream, buildVarsOverrides); } catch (IOException e) { @@ -195,21 +195,21 @@ public static Config load(File buildConfigFile, String buildVarsOverrides) { } } - public static Config load(InputStream configStream) { + public static PigConfiguration load(InputStream configStream) { return load(configStream, ""); } - public static Config load(InputStream configStream, String buildVarsOverrides) { + public static PigConfiguration load(InputStream configStream, String buildVarsOverrides) { if (buildVarsOverrides == null) { buildVarsOverrides = ""; } - Yaml yaml = new Yaml(new Constructor(Config.class)); + Yaml yaml = new Yaml(new Constructor(PigConfiguration.class)); try (InputStream in = preProcess(configStream, buildVarsOverrides)) { - Config config = (Config) yaml.load(in); - config.init(); - return config; + PigConfiguration pigConfiguration = (PigConfiguration) yaml.load(in); + pigConfiguration.init(); + return pigConfiguration; } catch (IOException e) { throw new RuntimeException("Unable to load config file", e); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java index 827e0b217..482fc4d63 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; /** * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
@@ -34,7 +36,16 @@ public class RepoGenerationData extends GenerationData { * list of groupId:artifactId:packaging:version with exact version */ private List externalAdditionalArtifacts = new ArrayList<>(); + private List excludeArtifacts = new ArrayList<>(); + private List sourceBuilds = new ArrayList<>(); private String bomGroupId; private String bomArtifactId; - private Boolean includeJavadoc; + private boolean includeJavadoc; + private boolean includeLicenses; + private boolean includeMavenMetadata; + private String buildScript; + private Set ignored; + private String additionalRepo; + private List> stages; + private Map parameters; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java index 29cf5a9bd..c9d156e96 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java @@ -22,5 +22,5 @@ * Date: 5/23/18 */ public enum RepoGenerationStrategy { - GENERATE, DOWNLOAD, IGNORE, PACK_ALL + GENERATE, DOWNLOAD, BUILD_CONFIGS, BUILD_GROUP, MILESTONE, IGNORE, PACK_ALL // DEPRECATED replaced by BUILD_CONFIGS } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java index 8093eef6c..fc2f27dba 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java @@ -21,7 +21,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import java.util.Collection; @@ -34,9 +34,10 @@ @AllArgsConstructor @NoArgsConstructor public class DataRoot { - private Config config; + private PigConfiguration pigConfiguration; private Deliverables deliverables; private String duplicates; private String repositoryContents; private Collection builds; + private String pncUrl; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java index e625a0dea..978063877 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java @@ -17,7 +17,8 @@ */ package org.jboss.pnc.bacon.pig.impl.documents; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.SharedContentReportGenerator; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -44,13 +45,17 @@ public class DocumentGenerator { private final String extrasPath; private final String releasePath; - private final Config config; + private final PigConfiguration pigConfiguration; private final Deliverables deliverables; - public DocumentGenerator(Config config, String releasePath, String extrasPath, Deliverables deliverables) { + public DocumentGenerator( + PigConfiguration pigConfiguration, + String releasePath, + String extrasPath, + Deliverables deliverables) { this.releasePath = releasePath; this.extrasPath = extrasPath; - this.config = config; + this.pigConfiguration = pigConfiguration; this.deliverables = deliverables; } @@ -67,7 +72,13 @@ public void generateDocuments(Map builds, RepositoryData repo) .sorted() .collect(Collectors.joining("\n")); - DataRoot templateData = new DataRoot(config, deliverables, duplicates, repositoryContents, builds.values()); + DataRoot templateData = new DataRoot( + pigConfiguration, + deliverables, + duplicates, + repositoryContents, + builds.values(), + Config.instance().getActiveProfile().getPnc().getUrl()); FileGenerator generator = new FileGenerator(Optional.empty()); log.debug("Generating documents with data: {}", templateData); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java index a4a939aaf..54bbd9afc 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java @@ -19,8 +19,6 @@ package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import com.redhat.red.build.finder.BuildConfig; import com.redhat.red.build.finder.BuildFinder; import com.redhat.red.build.finder.DistributionAnalyzer; @@ -30,12 +28,13 @@ import com.redhat.red.build.koji.model.json.util.KojiObjectMapper; import com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo; import com.redhat.red.build.koji.model.xmlrpc.KojiTagInfo; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; @@ -45,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.stream.Collectors; /** @@ -60,7 +60,7 @@ public class BrewSearcher { private static final String KOJI_BUILD_FINDER_CONFIG_PROP = "koji.build.finder.config"; - private static final String KOJI_BUILD_FINDER_CONFIG_RES = "koji-build-finder/config.json"; + private static final String KOJI_BUILD_FINDER_CONFIG_TEMPLATE = "koji-build-finder/config.json"; public static BuildConfig getKojiBuildFinderConfigFromFile(final File file) { try { @@ -78,14 +78,12 @@ public static BuildConfig getKojiBuildFinderConfigFromFile(final String filename } public static BuildConfig getKojiBuildFinderConfigFromResource(final String resourceName) { - final URL url = Resources.getResource(resourceName); + Properties props = new Properties(); + props.setProperty("KOJI_URL", Config.instance().getActiveProfile().getPig().getKojiHubUrl()); - try { - final String json = Resources.toString(url, Charsets.UTF_8); - return getKojiBuildFinderConfigFromJson(json); - } catch (IOException e) { - throw new IllegalStateException("Failed read Koji Build Finder configuration from resource: " + url, e); - } + String json = ResourceUtils.extractToStringWithFiltering(KOJI_BUILD_FINDER_CONFIG_TEMPLATE, props); + + return getKojiBuildFinderConfigFromJson(json); } public static BuildConfig getKojiBuildFinderConfigFromJson(final String json) { @@ -112,7 +110,7 @@ public static BuildConfig getKojiBuildFinderConfig() { return getKojiBuildFinderConfigFromFile(envFilename); } - return getKojiBuildFinderConfigFromResource(KOJI_BUILD_FINDER_CONFIG_RES); + return getKojiBuildFinderConfigFromResource(KOJI_BUILD_FINDER_CONFIG_TEMPLATE); } public static void fillBrewData(SharedContentReportRow row) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java index 82629b03e..c8c32e6bd 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java @@ -40,7 +40,8 @@ public class SharedContentReportGenerator { private static final Logger log = LoggerFactory.getLogger(SharedContentReportGenerator.class); - private static final String[] IGNORED = { "md5", "sha1", "-sources", "example-settings.xml", "README" }; + private static final String[] IGNORED = { "md5", "sha1", "-sources", "example-settings.xml", "README", + "maven-metadata.xml" }; private final Collection repositoryFiles; private final DASearcher daSearcher = new DASearcher(); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java index c8f0d8f9f..aeb402a22 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java @@ -24,10 +24,6 @@ import lombok.experimental.Delegate; import org.jboss.pnc.bacon.pig.impl.utils.GAV; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; - /** * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
* Date: 6/1/17 @@ -35,13 +31,12 @@ @Getter @Setter @ToString -public class CommunityDependency { +public class CommunityDependency implements CsvExportable { @Delegate - private final GAV gav; - private String recommendation; - private String availableVersions; - private DependencyState state; - private List usedForSwarm; + final GAV gav; + String recommendation; + String availableVersions; + DependencyState state; /* org/ow2/asm/asm-all/5.0.4/asm-all-5.0.4.jar */ public CommunityDependency(String logLine) { @@ -61,27 +56,18 @@ public String toPathSubstring() { return String.format("%s/%s/", getGroupId().replace('.', '/'), getArtifactId()); } - public void appendToCsv(FileWriter writer) { - try { - writer.append(toCsvLine()).append("\n"); - } catch (IOException e) { - throw new IllegalStateException("Cannot write to the output file", e); - } - } - /** * @return g:a:v; state; recommendation; availableVersions; usedForSwarm */ - private String toCsvLine() { + public String toCsvLine() { return String.format( - "%s:%s:%s; %s; %s; %s; %s", + "%s:%s:%s; %s; %s; %s", getGroupId(), getArtifactId(), getVersion(), state, recommendation, - availableVersions, - usedForSwarm); + availableVersions); } public org.jboss.da.model.rest.GAV toDaGav() { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java new file mode 100644 index 000000000..9c403b9fe --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java @@ -0,0 +1,37 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright 2017 Red Hat, Inc., and individual contributors + * as indicated by the @author tags. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da; + +import java.io.FileWriter; +import java.io.IOException; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +public interface CsvExportable { + String toCsvLine(); + + default void appendToCsv(FileWriter writer) { + try { + writer.append(toCsvLine()).append("\n"); + } catch (IOException e) { + throw new IllegalStateException("Cannot write to the output file", e); + } + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java new file mode 100644 index 000000000..b8f728fce --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java @@ -0,0 +1,35 @@ +package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da; + +import groovy.lang.Delegate; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +@Getter +@Setter +@ToString +public class ThorntailCommunityDependency implements CsvExportable { + + @Delegate + private final CommunityDependency communityDependency; + private List usedForThorntail; + + public ThorntailCommunityDependency(CommunityDependency communityDependency, List usedForThorntail) { + this.communityDependency = communityDependency; + this.usedForThorntail = usedForThorntail; + } + + public String toCsvLine() { + return String.format("%s; %s", communityDependency.toCsvLine(), usedForThorntail); + } + + public List getUsedForThorntail() { + return usedForThorntail; + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java index e6bdb7597..f12c5026b 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java @@ -22,17 +22,21 @@ import org.commonjava.maven.ext.cli.Cli; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.JavadocGenerationData; import org.jboss.pnc.bacon.pig.impl.config.JavadocGenerationStrategy; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Dependency; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Profile; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Project; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +46,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -74,10 +79,16 @@ public class JavadocManager extends DeliverableManager, Void> private File topLevelDirectory; private File archiveFile; private String scmRevision; + private boolean tempBuild; - public JavadocManager(Config config, String releasePath, Deliverables deliverables, Map builds) { - super(config, releasePath, deliverables, builds); - generationData = config.getFlow().getJavadocGeneration(); + public JavadocManager( + PigConfiguration pigConfiguration, + String releasePath, + Deliverables deliverables, + Map builds) { + super(pigConfiguration, releasePath, deliverables, builds); + this.tempBuild = PigProperties.get().isTemporary(); + this.generationData = pigConfiguration.getFlow().getJavadocGeneration(); } public void prepare() { @@ -93,6 +104,7 @@ public void prepare() { break; case IGNORE: log.info("Ignoring javadoc zip generation"); + deliverables.setJavadocZipName(null); break; default: throw new IllegalStateException("Unsupported javadoc generation strategy: " + strategy); @@ -106,7 +118,7 @@ protected void repackage(File contentsDirectory, File targetTopLevelDirectory) { @Override protected String getTargetTopLevelDirectoryName() { - return config.getTopLevelDirectoryPrefix() + "javadoc"; + return pigConfiguration.getTopLevelDirectoryPrefix() + "javadoc"; } @Override @@ -128,20 +140,26 @@ private boolean checkRequired() { } private void init() { - temporaryDestination = FileUtils.mkTempDir("javadoc"); - settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml").getAbsolutePath(); + this.temporaryDestination = FileUtils.mkTempDir("javadoc"); + if (this.tempBuild) { + this.settingsXml = ResourceUtils.extractToTmpFile("/indy-temp-settings.xml", "settings", ".xml") + .getAbsolutePath(); - localRepo = new File(temporaryDestination + File.separator + "localRepo"); - localRepo.mkdir(); - topLevelDirectory = new File(temporaryDestination, getTargetTopLevelDirectoryName()); - archiveFile = getTargetZipPath().toFile(); + } else { + this.settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml") + .getAbsolutePath(); + } + this.localRepo = new File(temporaryDestination + File.separator + "localRepo"); + this.localRepo.mkdir(); + this.topLevelDirectory = new File(temporaryDestination, getTargetTopLevelDirectoryName()); + this.archiveFile = getTargetZipPath().toFile(); - generationProject = generationData.getGenerationProject(); - sourceBuilds = generationData.getSourceBuilds(); - if (sourceBuilds == null || sourceBuilds.isEmpty()) { - sourceBuilds = builds.values().stream().map(PncBuild::getName).collect(Collectors.toList()); + this.generationProject = generationData.getGenerationProject(); + this.sourceBuilds = generationData.getSourceBuilds(); + if (this.sourceBuilds == null || this.sourceBuilds.isEmpty()) { + this.sourceBuilds = builds.values().stream().map(PncBuild::getName).collect(Collectors.toList()); } - scmRevision = generationData.getScmRevision(); + this.scmRevision = generationData.getScmRevision(); } private Collection findSourceBuilds() { @@ -264,9 +282,11 @@ private boolean runPME() { } } System.setOut(outStream); + log.info("PME Command run [{}]", cmd.toString()); if (new Cli().run(cmd.toString().split("\\s+")) != 0) { System.setOut(stdout); log.error("Error running PME see {}", filePath); + dumpLog(filePath); return false; } System.setOut(stdout); @@ -295,17 +315,24 @@ private boolean manipulateProject(Collection srcBuilds) { if (!writeProject(project)) { return false; } else { - runPME(); + return runPME(); } } - return true; + } + + private void dumpLog(String filePath) { + try { + System.out.println(new String(Files.readAllBytes(Paths.get(filePath)))); + } catch (IOException e) { + log.error("Unable to dump log {}", filePath, e); + } } private boolean executeMavenBuild() { log.debug("Executing Javadoc generation maven project"); String command = generationData.getBuildScript(); Process process = null; - if (command == null && command.isEmpty()) { + if (command == null || command.isEmpty()) { // Use a default mvn command on project command = "mvn package -B"; } @@ -320,13 +347,16 @@ private boolean executeMavenBuild() { process.waitFor(); if (process.exitValue() != 0) { log.error("Error while running Javadoc generation project [{}]", process.exitValue()); + dumpLog(mavenRun.getAbsolutePath()); return false; } } catch (IOException e) { log.error("Unable to start build Javadoc generation project", e); + dumpLog(mavenRun.getAbsolutePath()); return false; } catch (InterruptedException e) { log.error("Javadoc generation build was Interrupted", e); + dumpLog(mavenRun.getAbsolutePath()); return false; } return true; diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java index c6c0eb03f..5edb66712 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java @@ -22,15 +22,22 @@ import me.snowdrop.licenses.LicensesGeneratorException; import me.snowdrop.licenses.properties.GeneratorProperties; import me.snowdrop.licenses.utils.Gav; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.config.PigConfig; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.util.Collection; import java.util.List; +import java.util.Properties; import java.util.stream.Collectors; /** @@ -47,30 +54,68 @@ public static void generateLicenses(Collection gavs, File archiveFile, Stri File temporaryDestination = FileUtils.mkTempDir("licenses"); File topLevelDirectory = new File(temporaryDestination, topLevelDirectoryName); - generateLicenses(gavs, topLevelDirectory); + generateLicenses(gavs, topLevelDirectory, PigProperties.get().isTemporary()); FileUtils.zip(archiveFile, temporaryDestination, topLevelDirectory); log.debug("Generated zip archive {}", archiveFile); } - private static void generateLicenses(Collection gavs, File temporaryDestination) { + public static void generateLicenses(Collection gavs, File licensesDirectory, boolean useTempBuilds) { try { - LicensesGenerator generator = new LicensesGenerator(prepareGeneratorProperties()); + LicensesGenerator generator = new LicensesGenerator(prepareGeneratorProperties(useTempBuilds)); - generator.generateLicensesForGavs(gavsToLicenseGeneratorGavs(gavs), temporaryDestination.getAbsolutePath()); + generator.generateLicensesForGavs(gavsToLicenseGeneratorGavs(gavs), licensesDirectory.getAbsolutePath()); } catch (LicensesGeneratorException e) { throw new RuntimeException("Failed to generate licenses", e); } } + public static void extractLicenses(File repoZip, File archiveFile, String topLevelDirectoryName) { + File temporaryDestination = FileUtils.mkTempDir("licenses"); + File topLevelDirectory = new File(temporaryDestination, topLevelDirectoryName); + + try { + FileUtils.unzip(repoZip, topLevelDirectory, "^[^/]*/licenses/.*"); + File repoDir = Files.list(topLevelDirectory.toPath()).iterator().next().toFile(); + FileUtils.moveDirectoryContents(new File(repoDir, "licenses"), topLevelDirectory); + org.apache.commons.io.FileUtils.deleteDirectory(repoDir); + FileUtils.zip(archiveFile, temporaryDestination, topLevelDirectory); + log.info("Generated zip archive {}", archiveFile); + } catch (IOException ex) { + throw new RuntimeException("Failed to extract licenses from zip", ex); + } + + } + private static List gavsToLicenseGeneratorGavs(Collection gavs) { return gavs.stream() .map(gav -> new Gav(gav.getGroupId(), gav.getArtifactId(), gav.getVersion(), gav.getPackaging())) .collect(Collectors.toList()); } - private static GeneratorProperties prepareGeneratorProperties() { - File propertiesFile = ResourceUtils - .extractToTmpFile("/license-generator.properties", "license-generator", ".properties"); + private static GeneratorProperties prepareGeneratorProperties(boolean useTempBuilds) { + Properties props = new Properties(); + PigConfig pig = Config.instance().getActiveProfile().getPig(); + String licenseServiceUrl = pig.getLicenseServiceUrl(); + String licenseServiceProp = ""; + if (licenseServiceUrl != null) { + licenseServiceProp = String.format( + "licenseServiceUrl=http://%s/find-license-check-record-and-license-info", + licenseServiceUrl); + } + props.setProperty("licenseServiceUrl", licenseServiceProp); + if (useTempBuilds) { + props.setProperty("names", "Indy Temp Builds,Indy Static"); + props.setProperty("urls", Indy.getIndyTempUrl() + "," + Indy.getIndyUrl()); + } else { + props.setProperty("names", "Indy Static"); + props.setProperty("urls", Indy.getIndyUrl()); + } + + File propertiesFile = ResourceUtils.extractToTmpFileWithFiltering( + "/license-generator.properties", + "license-generator", + ".properties", + props); return new GeneratorProperties(propertiesFile.getAbsolutePath()); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java index 919a0241a..ddd7f3798 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java @@ -18,10 +18,12 @@ package org.jboss.pnc.bacon.pig.impl.license; import lombok.Getter; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.LicenseGenerationStrategy; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; +import org.jboss.pnc.bacon.pig.impl.config.RepoGenerationData; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.repo.RepositoryData; @@ -45,19 +47,25 @@ public class LicenseManager extends DeliverableManager, Void> @Getter private final GenerationData generationData; + private final boolean useTempBuilds; + public LicenseManager( - Config config, + PigConfiguration pigConfiguration, String releasePath, Deliverables deliverables, Map builds, RepositoryData repositoryData) { - super(config, releasePath, deliverables, builds); + super(pigConfiguration, releasePath, deliverables, builds); this.repositoryData = repositoryData; - generationData = config.getFlow().getLicensesGeneration(); + + generationData = pigConfiguration.getFlow().getLicensesGeneration(); + useTempBuilds = PigProperties.get().isTemporary(); } public void prepare() { - switch (generationData.getStrategy()) { + LicenseGenerationStrategy strategy = (generationData == null) ? LicenseGenerationStrategy.IGNORE + : generationData.getStrategy(); + switch (strategy) { case DOWNLOAD: downloadAndRepackage(); break; @@ -79,19 +87,27 @@ protected void repackage(File contentsDirectory, File targetTopLevelDirectory) { @Override protected String getTargetTopLevelDirectoryName() { - return config.getTopLevelDirectoryPrefix() + "licenses"; + return pigConfiguration.getTopLevelDirectoryPrefix() + "licenses"; } @Override protected Path getTargetZipPath() { - return Paths.get(releasePath + deliverables.getLicenseZipName()); + return Paths.get(releasePath, deliverables.getLicenseZipName()); } private void generate() { log.info("Generating licenses"); - LicenseGenerator.generateLicenses( - repositoryData.getGavs(), - getTargetZipPath().toFile(), - getTargetTopLevelDirectoryName()); + RepoGenerationData repoGen = pigConfiguration.getFlow().getRepositoryGeneration(); + if (repoGen.isIncludeLicenses()) { + LicenseGenerator.extractLicenses( + repositoryData.getRepositoryPath().toFile(), + getTargetZipPath().toFile(), + getTargetTopLevelDirectoryName()); + } else { + LicenseGenerator.generateLicenses( + repositoryData.getGavs(), + getTargetZipPath().toFile(), + getTargetTopLevelDirectoryName()); + } } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java index 76f54a9cd..13f14f8d9 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java @@ -48,10 +48,10 @@ public class BuildInfoCollector { private final BuildClient buildClient; private final BuildConfigurationClient buildConfigClient; - public void addDependencies(PncBuild bd) { + public void addDependencies(PncBuild bd, String filter) { List artifacts = null; try { - artifacts = toList(buildClient.getDependencyArtifacts(bd.getId())); + artifacts = toList(buildClient.getDependencyArtifacts(bd.getId(), Optional.empty(), Optional.of(filter))); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to get dependency artifacts for " + bd.getId(), e); } @@ -92,8 +92,8 @@ private String readLog(InputStream inputStream) throws IOException { StringBuilder logBuilder = new StringBuilder(); try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream); BufferedReader reader = new BufferedReader(inputStreamReader);) { - reader.lines().forEach(l -> logBuilder.append(l)); - return reader.toString(); + reader.lines().forEach(logBuilder::append); + return logBuilder.toString(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java index b1f227e9e..4dd25c8b0 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java @@ -19,7 +19,7 @@ import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.config.BuildConfig; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.config.ProductConfig; import org.jboss.pnc.bacon.pig.impl.utils.CollectionUtils; import org.jboss.pnc.bacon.pig.impl.utils.PncClientUtils; @@ -84,7 +84,7 @@ public class PncEntitiesImporter { private ProductMilestone milestone; private GroupConfiguration buildGroup; private List configs; - private Config config = PigContext.get().getConfig(); + private PigConfiguration pigConfiguration = PigContext.get().getPigConfiguration(); private PncConfigurator pncConfigurator = new PncConfigurator(); @@ -100,8 +100,10 @@ public PncEntitiesImporter() { public ImportResult performImport() { product = getOrGenerateProduct(); version = getOrGenerateVersion(); - milestone = pncConfigurator - .getOrGenerateMilestone(version, pncMilestoneString(), config.getProduct().getIssueTrackerUrl()); + milestone = pncConfigurator.getOrGenerateMilestone( + version, + pncMilestoneString(), + pigConfiguration.getProduct().getIssueTrackerUrl()); pncConfigurator.markMilestoneCurrent(version, milestone); buildGroup = getOrGenerateBuildGroup(); @@ -222,8 +224,8 @@ private void addConfigurationToGroup(String newConfigId) { private List getAddOrUpdateBuildConfigs() { log.info("Adding/updating build configurations"); List currentConfigs = getCurrentBuildConfigs(); - dropConfigsFromInvalidVersion(currentConfigs, config.getBuilds()); - return updateOrCreate(currentConfigs, config.getBuilds()); + dropConfigsFromInvalidVersion(currentConfigs, pigConfiguration.getBuilds()); + return updateOrCreate(currentConfigs, pigConfiguration.getBuilds()); } private Optional getBuildConfigFromName(String name) { @@ -439,8 +441,9 @@ private GroupConfiguration getOrGenerateBuildGroup() { private Optional getBuildGroup() { try { - return toStream(groupConfigClient.getAll(empty(), Optional.of("name=='" + config.getGroup() + "'"))) - .findAny(); + return toStream( + groupConfigClient.getAll(empty(), Optional.of("name=='" + pigConfiguration.getGroup() + "'"))) + .findAny(); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to check if build group exists"); } @@ -455,7 +458,7 @@ private Product getOrGenerateProduct() { } private Optional getProduct() { - String productName = config.getProduct().getName(); + String productName = pigConfiguration.getProduct().getName(); try { return maybeSingle(productClient.getAll(empty(), findByNameQuery(productName))); } catch (RemoteResourceException e) { @@ -466,7 +469,7 @@ private Optional getProduct() { private ProductVersion generateVersion() { ProductVersion productVersion = ProductVersion.builder() .product(product) - .version(config.getMajorMinor()) + .version(pigConfiguration.getMajorMinor()) .build(); try { return versionClient.createNew(productVersion); @@ -476,7 +479,7 @@ private ProductVersion generateVersion() { } private Product generateProduct() { - ProductConfig productConfig = config.getProduct(); + ProductConfig productConfig = pigConfiguration.getProduct(); Product product = Product.builder() .name(productConfig.getName()) .abbreviation(productConfig.getAbbreviation()) @@ -489,25 +492,28 @@ private Product generateProduct() { } private GroupConfiguration generateBuildGroup(ProductVersionRef version) { - GroupConfiguration group = GroupConfiguration.builder().productVersion(version).name(config.getGroup()).build(); + GroupConfiguration group = GroupConfiguration.builder() + .productVersion(version) + .name(pigConfiguration.getGroup()) + .build(); try { return groupConfigClient.createNew(group); } catch (ClientException e) { - throw new RuntimeException("Failed to create group config: " + config.getGroup()); + throw new RuntimeException("Failed to create group config: " + pigConfiguration.getGroup()); } } public ImportResult readCurrentPncEntities() { - product = getProduct() - .orElseThrow(() -> new RuntimeException("Unable to product " + config.getProduct().getName())); + product = getProduct().orElseThrow( + () -> new RuntimeException("Unable to product " + pigConfiguration.getProduct().getName())); version = getVersion().orElseThrow( () -> new RuntimeException( - "Unable to find version " + config.getMajorMinor() + " for product " + product)); + "Unable to find version " + pigConfiguration.getMajorMinor() + " for product " + product)); milestone = pncConfigurator.getExistingMilestone(version, pncMilestoneString()) .orElseThrow(() -> new RuntimeException("Unable to find milestone " + pncMilestoneString())); // TODO buildGroup = getBuildGroup() - .orElseThrow(() -> new RuntimeException("Unable to find build group " + config.getGroup())); + .orElseThrow(() -> new RuntimeException("Unable to find build group " + pigConfiguration.getGroup())); configs = getBuildConfigs(); @@ -526,7 +532,7 @@ private List getBuildConfigs() { private Optional getVersion() { try { - Optional byName = query("version=='%s'", config.getMajorMinor()); + Optional byName = query("version=='%s'", pigConfiguration.getMajorMinor()); return maybeSingle(productClient.getProductVersions(product.getId(), empty(), byName)); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to query for version", e); @@ -534,6 +540,7 @@ private Optional getVersion() { } private String pncMilestoneString() { - return config.getMajorMinor() + "." + config.getMicro() + "." + config.getMilestone(); + return pigConfiguration.getMajorMinor() + "." + pigConfiguration.getMicro() + "." + + pigConfiguration.getMilestone(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java new file mode 100644 index 000000000..7d6dc1643 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java @@ -0,0 +1,61 @@ +package org.jboss.pnc.bacon.pig.impl.repo; + +import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; + +import java.util.Map; +import java.util.Optional; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +public class ArtifactVersion { + public static String prefix = "artifactVersion://"; + + private ArtifactVersion() { + } + + /* + * gets artifact coordinates from a location of a form artifactVersion://groupId:artifactId[@buildName] and gets a + * version of the artifact from the build + */ + public static String get(String location, String defaultBuildName, Map builds) { + if (!location.startsWith(ArtifactVersion.prefix)) { + throw new RuntimeException("location: " + location + " is not a proper artifactVersion location"); + } + String value = location.substring(prefix.length()); + + String[] split = value.split("@"); + + String gaString = split[0]; + String[] gaSplit = gaString.split(":"); + if (gaSplit.length != 2) { + throw new RuntimeException( + "Expected artifactVersion://groupId:artifactId... as the version locator, got: " + value); + } + + String groupId = gaSplit[0]; + String artifactId = gaSplit[1]; + + String buildName = split.length > 1 ? split[1] : defaultBuildName; + + PncBuild pncBuild = builds.get(buildName); + if (pncBuild == null) { + throw new RuntimeException("Build " + buildName + " not found among the builds"); + } + + Optional maybeArtifact = pncBuild.getBuiltArtifacts().stream().filter(a -> { + GAV gav = a.toGAV(); + return gav.getArtifactId().equals(artifactId) && gav.getGroupId().equals(groupId); + }).findAny(); + + return maybeArtifact + .orElseThrow( + () -> new RuntimeException( + "Unable to find artifact matching " + groupId + ":" + artifactId + + " in artifacts produced by " + buildName)) + .toGAV() + .getVersion(); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java index a2a4a20ba..8e2b901af 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java @@ -19,6 +19,9 @@ import org.jboss.pnc.bacon.pig.impl.utils.FileDownloadUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.net.URI; @@ -30,36 +33,38 @@ */ public class ExternalArtifactDownloader { + private static final Logger log = LoggerFactory.getLogger(ExternalArtifactDownloader.class); + private ExternalArtifactDownloader() { } - private static final String TEMPORARY_ARTIFACTS_URL; - private static final String ARTIFACTS_URL; + public static File downloadExternalArtifact(GAV gav, Path targetRepoContents, boolean sourcesOptional) { + File targetPath = targetPath(gav, targetRepoContents); - static { - String indyRepoUrl = System.getenv("INDY_REPO_URL"); - if (indyRepoUrl == null) { - indyRepoUrl = "http://indy.psi.redhat.com/"; - } - if (!indyRepoUrl.endsWith("/")) { - indyRepoUrl = indyRepoUrl + "/"; - } - TEMPORARY_ARTIFACTS_URL = indyRepoUrl + "api/content/maven/group/temporary-builds"; - ARTIFACTS_URL = indyRepoUrl + "api/group/builds-untested+shared-imports+public"; + return downloadExternalArtifact(gav, targetPath, sourcesOptional); } - public static File downloadExternalArtifact(GAV gav, Path targetRepoContents) { - Path versionPath = targetRepoContents.resolve(gav.toVersionPath()); - versionPath.toFile().mkdirs(); - - File targetPath = versionPath.resolve(gav.toFileName()).toFile(); + public static File downloadExternalArtifact(GAV gav, File targetPath, boolean sourcesOptional) { + targetPath.toPath().getParent().toFile().mkdirs(); - String indyUrl = gav.isTemporary() ? TEMPORARY_ARTIFACTS_URL : ARTIFACTS_URL; + String indyUrl = gav.isTemporary() ? Indy.getIndyTempUrl() : Indy.getIndyUrl(); URI downloadUrl = URI.create(String.format("%s/%s", indyUrl, gav.toUri())); - - FileDownloadUtils.downloadTo(downloadUrl, targetPath); + try { + FileDownloadUtils.downloadTo(downloadUrl, targetPath); + } catch (RuntimeException any) { + if (sourcesOptional && "sources".equals(gav.getClassifier()) || "javadoc".equals(gav.getClassifier())) { + log.warn("Unable to download sources for " + gav, any); + } else { + throw any; + } + } return targetPath; } + + public static File targetPath(GAV gav, Path targetRepoContents) { + Path versionPath = targetRepoContents.resolve(gav.toVersionPath()); + return versionPath.resolve(gav.toFileName()).toFile(); + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java index 4afa4cb3e..b81800276 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java @@ -1,7 +1,6 @@ package org.jboss.pnc.bacon.pig.impl.repo; -import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; -import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -21,8 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; +import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; /** * @author Ken Finnigan @@ -31,7 +30,8 @@ public class ParentPomDownloader { public static final Logger log = LoggerFactory.getLogger(ParentPomDownloader.class); - private ParentPomDownloader() { + private ParentPomDownloader(Path repoPath) { + this.repoPath = repoPath; } public static void addParentPoms(Path repoPath) { @@ -40,61 +40,38 @@ public static void addParentPoms(Path repoPath) { } try { - new ParentPomDownloader().process(repoPath); + new ParentPomDownloader(repoPath).process(); } catch (IOException e) { throw new RuntimeException("Unable to download parent poms", e); } } - private void process(final Path repoPath) throws IOException { - Set pomArtifacts = retrievePoms(repoPath); - - File execDir = FileUtils.mkTempDir("parent-pom-retrieval"); - - pomArtifacts.stream().filter(Pom::hasParent).forEach(p -> processPoms(repoPath, p)); - - String settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml").getAbsolutePath(); - toDownload.parallelStream().forEach(gav -> { - log.debug("Downloading: {}", gav.toString()); - // Call Maven to download dependency - - ProcessBuilder builder = new ProcessBuilder( - "mvn", - "-B", - "org.apache.maven.plugins:maven-dependency-plugin:3.0.1:get", - "-Dartifact=" + gav.toString(), - "-Dmaven.repo.local=" + repoPath.toAbsolutePath().toString(), - "-s", - settingsXml); - - builder.directory(execDir).inheritIO(); + private void process() throws IOException { + // iterate until the working set is empty + // this is to download parent POM of a parent POM of a parent POM of a parent POM of a ... + do { + toDownload.clear(); + doProcess(); + } while (!toDownload.isEmpty()); + } - Process process = null; - try { - process = builder.start(); - } catch (IOException e) { - log.error("Unable to download gav {}", gav.toString(), e); - System.out.println(14); - } + private void doProcess() throws IOException { + Files.walk(repoPath) + .filter(Files::isRegularFile) + .filter(ParentPomDownloader::isPom) + .map(Pom::new) + .filter(Pom::hasParent) + .forEach(p -> processPom(repoPath, p)); - while (process.isAlive()) { - try { - Thread.sleep(1000); - if (process.exitValue() == 0) { - break; - } - } catch (IllegalThreadStateException e) { - // ignore as process not exited - } catch (InterruptedException e) { - break; - } - } - } + toDownload.parallelStream().map(PomGAV::toGav).forEach(this::download); + } - ); + private void download(GAV gav) { + log.info("Downloading missing parent POM {}", gav); + ExternalArtifactDownloader.downloadExternalArtifact(gav, repoPath, false); } - private void processPoms(Path repoPath, Pom pom) { + private void processPom(Path repoPath, Pom pom) { PomGAV coords = parentCoordinates(pom); if (!coords.version.contains("redhat")) { // community parent POM not required @@ -102,11 +79,12 @@ private void processPoms(Path repoPath, Pom pom) { } Path parentDir = artifactDir(repoPath, coords); - Path parentPomPath = parentDir.resolve(coords.artifactId() + "-" + coords.version() + ".pom"); + Path parentPomPath = parentDir.resolve(coords.artifactId + "-" + coords.version + ".pom"); if (!alreadyChecked.contains(coords)) { if (!Files.isRegularFile(parentPomPath)) { // File missing + log.debug("Will download {} because it's a parent of {}", coords, pom.path); toDownload.add(coords); } @@ -114,22 +92,14 @@ private void processPoms(Path repoPath, Pom pom) { } } - private Set retrievePoms(Path repoPath) throws IOException { - return Files.walk(repoPath) - .filter(Files::isRegularFile) - .filter(ParentPomDownloader::isPom) - .map(Pom::new) - .collect(Collectors.toSet()); - } - private static boolean isPom(Path path) { return path.toString().endsWith(".pom"); } private static Path artifactDir(Path repoPath, PomGAV coords) { - Path groupDir = repoPath.resolve(coords.groupId().replace('.', File.separatorChar)); - Path artifactDir = groupDir.resolve(coords.artifactId()); - return artifactDir.resolve(coords.version()); + Path groupDir = repoPath.resolve(coords.groupId.replace('.', File.separatorChar)); + Path artifactDir = groupDir.resolve(coords.artifactId); + return artifactDir.resolve(coords.version); } private PomGAV parentCoordinates(Pom pom) { @@ -174,9 +144,11 @@ private XPathExpression parentVersionExpression() throws XPathExpressionExceptio return parentVersionExpression; } - private Set alreadyChecked = new HashSet<>(); + private final Set alreadyChecked = new HashSet<>(); + + private final Set toDownload = new HashSet<>(); - private Set toDownload = new HashSet<>(); + private final Path repoPath; private XPath xpath; @@ -186,7 +158,7 @@ private XPathExpression parentVersionExpression() throws XPathExpressionExceptio private XPathExpression parentVersionExpression; - private class Pom { + private static class Pom { private final Path path; private DocumentBuilder documentBuilder; @@ -220,7 +192,7 @@ private DocumentBuilder documentBuilder() throws ParserConfigurationException { } } - private class PomGAV { + private static class PomGAV { private final String groupId; private final String artifactId; @@ -233,21 +205,29 @@ private class PomGAV { this.version = version; } - String groupId() { - return groupId; + @Override + public String toString() { + return groupId + ':' + artifactId + ':' + version + ":pom"; } - String artifactId() { - return artifactId; + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof PomGAV)) + return false; + PomGAV pomGAV = (PomGAV) o; + return Objects.equals(groupId, pomGAV.groupId) && Objects.equals(artifactId, pomGAV.artifactId) + && Objects.equals(version, pomGAV.version); } - String version() { - return version; + @Override + public int hashCode() { + return Objects.hash(groupId, artifactId, version); } - @Override - public String toString() { - return groupId + ':' + artifactId + ':' + version + ":pom"; + public GAV toGav() { + return new GAV(groupId, artifactId, version, "pom"); } } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java index 3f5428712..d5b3d8aa0 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java @@ -20,12 +20,15 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.config.RepoGenerationData; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.OSCommandExecutor; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.jboss.pnc.bacon.pig.impl.utils.XmlUtils; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; @@ -36,8 +39,12 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.apache.commons.lang3.StringUtils.isEmpty; + /** * TODO: replace with programmatic maven api to remove OSCommandExecutor? TODO: CON: would mean that users have to stick * to the same maven version @@ -50,46 +57,95 @@ public class RepoBuilder { private static final String POM_TEMPLATE_LOCATION = "/pom-template.xml"; private static final String ENCODING = "UTF-8"; - private final Config config; + private final PigConfiguration pigConfiguration; private final boolean removeGeneratedM2Dups; + private final boolean tempBuild; + private final RepoGenerationData repoGeneration; + private final Map builds; + private final String additionalRepo; String topLevelDirectoryName; Path configurationDirectory; - public RepoBuilder(Config config, Path configurationDirectory, boolean removeGeneratedM2Dups) { - this.config = config; - topLevelDirectoryName = config.getTopLevelDirectoryPrefix() + "maven-repository"; + public RepoBuilder( + PigConfiguration pigConfiguration, + String additionalRepo, + Path configurationDirectory, + Map builds) { + this.pigConfiguration = pigConfiguration; + this.additionalRepo = additionalRepo; + this.builds = builds; this.configurationDirectory = configurationDirectory; - this.removeGeneratedM2Dups = removeGeneratedM2Dups; + + repoGeneration = pigConfiguration.getFlow().getRepositoryGeneration(); + removeGeneratedM2Dups = PigProperties.get().isRemoveGeneratedM2Dups(); + tempBuild = PigProperties.get().isTemporary(); + topLevelDirectoryName = pigConfiguration.getTopLevelDirectoryPrefix() + "maven-repository"; } - public File build(File bomFile) { + public void build(File bomFile, File repoParentDir, Predicate artifactSelector) { try { - File projectLocation = createProject(bomFile); - String settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml") - .getAbsolutePath(); - File m2Repo = buildProjectWithOverriddenM2(projectLocation, settingsXml); - RepositoryUtils.removeIrrelevantFiles(m2Repo); + createAndBuildProject(bomFile, repoParentDir, artifactSelector); + RepositoryUtils.removeIrrelevantFiles(repoParentDir); if (removeGeneratedM2Dups) { - RepositoryUtils.keepOnlyLatestRedHatArtifacts(m2Repo); + RepositoryUtils.keepOnlyLatestRedHatArtifacts(repoParentDir); } - return m2Repo; } catch (IOException e) { throw new RuntimeException("Unable to build pom", e); } } - private File buildProjectWithOverriddenM2(File projectLocation, String settingsXml) { + private void createAndBuildProject(File bomFile, File repoParentDir, Predicate artifactSelector) + throws IOException { + File projectLocation = createProject(bomFile, artifactSelector); + String indySettings; + if (tempBuild) { + indySettings = "/indy-temp-settings.xml"; + } else { + indySettings = "/indy-settings.xml"; + } + String settingsXml = ResourceUtils + .extractToTmpFileWithFiltering( + indySettings, + "settings", + ".xml", + settingsProps(tempBuild, additionalRepo)) + .getAbsolutePath(); + + File repoDir = new File(repoParentDir, RepoDescriptor.MAVEN_REPOSITORY); + + buildProjectWithOverriddenM2(projectLocation, repoDir, settingsXml); + } + + private Properties settingsProps(boolean tempBuild, String additionalRepo) { + Properties result = new Properties(); + if (tempBuild) { + String repoDef = "--> \n" + " additional\n" + " " + + additionalRepo + "\n" + " \n" + " additional-plugins\n" + + " " + additionalRepo + "\n" + " + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + indy-repositories + + + indy + ${INDY_URL} + + + + + indy + ${INDY_URL} + + + + + + indy-repositories + maven-central + + diff --git a/pig/src/main/resources/indy-temp-settings.xml b/pig/src/main/resources/indy-temp-settings.xml new file mode 100644 index 000000000..8cb2594a4 --- /dev/null +++ b/pig/src/main/resources/indy-temp-settings.xml @@ -0,0 +1,69 @@ + + + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + indy-repositories + + + indy + ${INDY_URL} + + + indy-temp + ${INDY_TMP_URL}/ + + + + + + indy + ${INDY_URL} + + + indy-temp + ${INDY_TMP_URL} + + + + + + + + + indy-repositories + maven-central + + + diff --git a/pig/src/main/resources/koji-build-finder/config.json b/pig/src/main/resources/koji-build-finder/config.json new file mode 100644 index 000000000..631bbfa91 --- /dev/null +++ b/pig/src/main/resources/koji-build-finder/config.json @@ -0,0 +1,10 @@ +{ + "archive-extensions" : [ "dll", "dylib", "ear", "jar", "jdocbook", "jdocbook-style", "kar", "plugin", "pom", "rar", "sar", "so", "war", "xml" ], + "archive-types" : [ "jar", "xml", "pom", "so", "dll", "dylib" ], + "checksum-only" : false, + "checksum-type" : "md5", + "excludes" : [ "^(?!.*/pom\\.xml$).*/.*\\.xml$" ], + "koji-hub-url" : "${KOJI_URL}/brewhub", + "koji-web-url" : "${KOJI_URL}/brew" +} + diff --git a/pig/src/main/resources/license-generator.properties b/pig/src/main/resources/license-generator.properties new file mode 100644 index 000000000..08635de94 --- /dev/null +++ b/pig/src/main/resources/license-generator.properties @@ -0,0 +1,4 @@ +# to run with PNC stage, you need to set the `INDY_REPO_URL` environment variable to http://indy-stage.psi.redhat.com/ +repository.names=${names} +repository.urls=${urls} +$licenseServiceUrl \ No newline at end of file diff --git a/pig/src/main/resources/pom-template.xml b/pig/src/main/resources/pom-template.xml new file mode 100644 index 000000000..a3de5e71b --- /dev/null +++ b/pig/src/main/resources/pom-template.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + com.example + user-project + 1.0-SNAPSHOT + + + + + + + + import + pom + + + + + + + + + + maven-dependency-plugin + org.apache.maven.plugins + 2.10 + + + download-sources + prepare-package + + sources + + + + + + + + + 1.8 + 1.8 + + diff --git a/pig/src/main/resources/release.sh b/pig/src/main/resources/release.sh new file mode 100644 index 000000000..2f4cfd7f5 --- /dev/null +++ b/pig/src/main/resources/release.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e +# mstodo: + +<#noparse> +function getPushStatus() { + pnc brew-push status $1 | grep "status" | cut -d : -f 2 | tr -d '[:space:]' | tr -d '"' +} + +function waitForPush() { + while true + do + status=$(getPushStatus $1) + if [ "${status}" == "SUCCESS" ] + then + break; + fi + + if [ "${status}" == "SYSTEM_ERROR" ] || [ "${status}" == "CANCELED" ] || [ "${status}" == "FAILED" ] + then + echo "Failed to push build $1 to Brew" + exit 1 + fi + echo "Waiting for build $1 to be pushed to Brew" + sleep 5 + done +} + + +<#list buildsToPush as build> +pnc brew-push build ${build} ${brewTag} +waitForPush ${build} + + +/bin/bash ${nvrListScriptLocation} ${repoZipLocation} ${targetPath} ${kojiHubUrl} + +pnc close-milestone ${milestoneId} --wait diff --git a/pig/src/main/resources/repository-README.md b/pig/src/main/resources/repository-README.md new file mode 100644 index 000000000..804179897 --- /dev/null +++ b/pig/src/main/resources/repository-README.md @@ -0,0 +1,4 @@ +${PRODUCT_NAME} Maven Repository +======================= + +This archive contains Maven repository artifacts for ${PRODUCT_NAME}. diff --git a/pig/src/main/resources/repository-example-settings.xml b/pig/src/main/resources/repository-example-settings.xml new file mode 100644 index 000000000..5c5154d93 --- /dev/null +++ b/pig/src/main/resources/repository-example-settings.xml @@ -0,0 +1,54 @@ + + + + + + + repository + + + repository + file:///path/to/repo/repository/maven-repository + + true + + + false + + + + + + repository + file:///path/to/repo/repository/maven-repository + + true + + + false + + + + + + + + repository + + \ No newline at end of file diff --git a/pig/src/main/resources/rh-license-exceptions.json b/pig/src/main/resources/rh-license-exceptions.json new file mode 100644 index 000000000..af1fbc80e --- /dev/null +++ b/pig/src/main/resources/rh-license-exceptions.json @@ -0,0 +1,1113 @@ +[ + { + "groupId": "org.apache.tomcat", + "artifactId": "servlet-api", + "version": "6.0.41", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + { + "name": "Common Development And Distribution License (CDDL) Version 1.0", + "url": "http://www.opensource.org/licenses/cddl1.txt" + } + ] + }, + { + "groupId": "com.squareup.okhttp3", + "artifactId": "benchmarks", + "version": "3.9.0.redhat-3", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.tomcat", + "artifactId": "servlet-api", + "version": "6.0.41-redhat", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + { + "name": "Common Development And Distribution License (CDDL) Version 1.0", + "url": "http://www.opensource.org/licenses/cddl1.txt" + } + ] + }, + { + "groupId": "antlr", + "artifactId": "antlr", + "version": "2.7.7", + "licenses": [ + { + "name": "The Antlr 2.7.7 License", + "url": "http://www.antlr2.org/license.html" + } + ] + }, + { + "groupId": "antlr", + "artifactId": "antlr", + "version": "2.7.7-redhat", + "licenses": [ + { + "name": "The Antlr 2.7.7 License", + "url": "http://www.antlr2.org/license.html" + } + ] + }, + { + "groupId": "dom4j", + "artifactId": "dom4j", + "version": "1.6.1", + "licenses": [ + { + "name": "Plexus Classworlds License", + "url": "https://raw.githubusercontent.com/dom4j/dom4j/dom4j_1_6_1/LICENSE.txt" + } + ] + }, + { + "groupId": "dom4j", + "artifactId": "dom4j", + "version": "1.6.1-redhat", + "licenses": [ + { + "name": "Plexus Classworlds License", + "url": "https://raw.githubusercontent.com/dom4j/dom4j/dom4j_1_6_1/LICENSE.txt" + } + ] + }, + { + "groupId": "com.h2database", + "artifactId": "h2", + "version": "1.4.196", + "licenses": [ + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + }, + { + "name": "Mozilla Public License v2.0", + "url": "http://www.mozilla.org/MPL/2.0" + } + ] + }, + { + "groupId": "org.jboss.openjdk-orb", + "artifactId": "openjdk-orb", + "version": "8.0.8.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.rmi", + "artifactId": "jboss-rmi-api_1.0_spec", + "version": "1.0.5.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ejb", + "artifactId": "jboss-ejb-api_3.2_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.2_spec", + "version": "2.2.13.redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.interceptor", + "artifactId": "jboss-interceptors-api_1.2_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.jms", + "artifactId": "jboss-jms-api_2.0_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.resource", + "artifactId": "jboss-connector-api_1.7_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.security.jacc", + "artifactId": "jboss-jacc-api_1.5_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.transaction", + "artifactId": "jboss-transaction-api_1.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ws.rs", + "artifactId": "jboss-jaxrs-api_2.0_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.bind", + "artifactId": "jboss-jaxb-api_2.2_spec", + "version": "1.0.4.Final-redhat-4", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.rpc", + "artifactId": "jboss-jaxrpc-api_1.1_spec", + "version": "1.0.1.Final-redhat-5", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.soap", + "artifactId": "jboss-saaj-api_1.3_spec", + "version": "1.0.4.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.ws", + "artifactId": "jboss-jaxws-api_2.2_spec", + "version": "2.0.4.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.2_spec", + "version": "2.2.13.SP2-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.4.redhat-7", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.4.redhat-7", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.reactivex", + "artifactId": "rxjava", + "version": "1.3.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.jboss.openjdk-orb", + "artifactId": "openjdk-orb", + "version": "8.1.2.Final-redhat-00001", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ejb", + "artifactId": "jboss-ejb-api_3.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.3_spec", + "version": "2.3.5.SP1-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.interceptor", + "artifactId": "jboss-interceptors-api_1.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.jms", + "artifactId": "jboss-jms-api_2.0_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.resource", + "artifactId": "jboss-connector-api_1.7_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.security.jacc", + "artifactId": "jboss-jacc-api_1.5_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.transaction", + "artifactId": "jboss-transaction-api_1.2_spec", + "version": "1.1.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ws.rs", + "artifactId": "jboss-jaxrs-api_2.1_spec", + "version": "1.0.1.Final-redhat-00001", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.bind", + "artifactId": "jboss-jaxb-api_2.3_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.rpc", + "artifactId": "jboss-jaxrpc-api_1.1_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.soap", + "artifactId": "jboss-saaj-api_1.3_spec", + "version": "1.0.6.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.ws", + "artifactId": "jboss-jaxws-api_2.3_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.apache.ant", + "artifactId": "ant-parent", + "version": "1.9.4.redhat-1", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.ant", + "artifactId": "ant-launcher", + "version": "1.9.4.redhat-1", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.10.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.10.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.1.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.1.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.2.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.2.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.3.0.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.3.0.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.3.0.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.3.0.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "12.0.0.redhat-00002", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "13.0.0.redhat-00001", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "14.0.0.redhat-00001", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.fasterxml.jackson.module", + "artifactId": "jackson-module-scala_2.12", + "version": "2.9.9.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "com.fasterxml.jackson.module", + "artifactId": "jackson-module-scala_2.12", + "version": "2.9.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.reactivestreams", + "artifactId": "reactive-streams", + "version-regexp": ".*", + "licenses": [ + { + "name": "Creative Commons Zero 1.0 Universal", + "url": "http://creativecommons.org/publicdomain/zero/1.0/legalcode" + } + ] + }, + { + "groupId": "com.thoughtworks.paranamer", + "artifactId": "paranamer-parent", + "version": "2.8.0.redhat-00001", + "licenses": [ + { + "name": "BSD 3-clause \"New\" or \"Revised\" License", + "url": "https://opensource.org/licenses/BSD-3-Clause" + } + ] + }, + { + "groupId": "com.thoughtworks.paranamer", + "artifactId": "paranamer", + "version": "2.8.0.redhat-00001", + "licenses": [ + { + "name": "BSD 3-clause \"New\" or \"Revised\" License", + "url": "https://opensource.org/licenses/BSD-3-Clause" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "btf", + "version": "1.2.0.redhat-00006", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "jackson-coreutils", + "version": "1.6.0.redhat-00006", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "msg-simple", + "version": "1.1.0.redhat-00005", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "io.reactivex.rxjava2", + "artifactId": "rxjava", + "version": "2.2.18.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "javax.persistence", + "artifactId": "javax.persistence-api", + "version": "2.2.0.redhat-1", + "licenses": [ + { + "name": "Eclipse Distribution License, Version 1.0", + "url": "https://repository.jboss.org/licenses/edl-1.0.txt" + }, + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-client", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-core", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-thrift", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-tracerresolver", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.eclipse", + "artifactId": "yasson", + "version-regexp": ".*", + "licenses": [ + { + "name": "Eclipse Distribution License, Version 1.0", + "url": "https://repository.jboss.org/licenses/edl-1.0.txt" + }, + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + }, + { + "groupId": "org.eclipse.jdt.core.compiler", + "artifactId": "ecj", + "version-range": "[4.0,5.0)", + "licenses": [ + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + } +] diff --git a/pig/src/main/resources/settings-template.xml b/pig/src/main/resources/settings-template.xml new file mode 100644 index 000000000..14cb3c5f6 --- /dev/null +++ b/pig/src/main/resources/settings-template.xml @@ -0,0 +1,57 @@ + + + + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + additional-repo + + + additional-repo + ADD_REPO_URL + + + + + additional-repo + ADD_REPO_URL + + + + + + additional-repo + maven-central + + From f109b6454f2a37912563f07090f66bb66f5cfb19 Mon Sep 17 00:00:00 2001 From: Michal Szynkiewicz Date: Fri, 5 Jun 2020 18:06:35 +0200 Subject: [PATCH 2/5] some pork --- README.md | 8 +- .../pnc/bacon/common/cli/AbstractCommand.java | 8 +- .../jboss/pnc/bacon/config/ConfigProfile.java | 1 + .../org/jboss/pnc/bacon/config/PigConfig.java | 13 + .../org/jboss/pnc/bacon/test/CLIExecutor.java | 7 +- .../org/jboss/pnc/bacon/test/pig/PigTest.java | 49 +- .../config/{pig.yaml => build-config.yaml} | 0 .../java/org/jboss/pnc/bacon/pig/Pig.java | 17 +- .../org/jboss/pnc/bacon/pig/PigFacade.java | 29 +- .../jboss/pnc/bacon/pig/impl/PigContext.java | 34 +- .../pnc/bacon/pig/impl/PigProperties.java | 37 + .../pnc/bacon/pig/impl/addons/AddOn.java | 18 +- .../bacon/pig/impl/addons/AddOnFactory.java | 23 +- .../addons/ExtraDeliverableDownloader.java | 17 +- .../BuildLogWithDependencyTrees.java | 88 ++ ...roProfileSmallRyeCommunityDepAnalyzer.java | 63 + .../quarkus/QuarkusCommunityDepAnalyzer.java | 105 +- .../addons/runtime/CommunityDepAnalyzer.java | 50 +- .../addons/runtime/RepoBuildLogProcessor.java | 67 - .../runtime/RuntimeDependenciesAnalyzer.java | 18 +- .../addons/runtime/ThorntailDepAnalyzer.java | 42 + .../impl/addons/spring/BomVerifierAddon.java | 18 +- .../NotYetAlignedFromDependencyTree.java | 6 +- .../pig/impl/common/DeliverableManager.java | 8 +- .../bacon/pig/impl/config/BuildConfig.java | 45 +- .../{Config.java => PigConfiguration.java} | 20 +- .../pig/impl/config/RepoGenerationData.java | 13 +- .../impl/config/RepoGenerationStrategy.java | 2 +- .../bacon/pig/impl/documents/DataRoot.java | 5 +- .../pig/impl/documents/DocumentGenerator.java | 21 +- .../documents/sharedcontent/BrewSearcher.java | 22 +- .../SharedContentReportGenerator.java | 3 +- .../sharedcontent/da/CommunityDependency.java | 30 +- .../sharedcontent/da/CsvExportable.java | 37 + .../da/ThorntailCommunityDependency.java | 35 + .../pig/impl/javadoc/JavadocManager.java | 68 +- .../pig/impl/license/LicenseGenerator.java | 57 +- .../pig/impl/license/LicenseManager.java | 38 +- .../pig/impl/pnc/BuildInfoCollector.java | 8 +- .../pig/impl/pnc/PncEntitiesImporter.java | 45 +- .../bacon/pig/impl/repo/ArtifactVersion.java | 61 + .../impl/repo/ExternalArtifactDownloader.java | 45 +- .../pig/impl/repo/ParentPomDownloader.java | 128 +- .../pnc/bacon/pig/impl/repo/RepoBuilder.java | 141 ++- .../bacon/pig/impl/repo/RepoDescriptor.java | 2 +- .../pnc/bacon/pig/impl/repo/RepoManager.java | 201 ++- .../bacon/pig/impl/repo/RepositoryUtils.java | 83 ++ .../pig/impl/script/ScriptGenerator.java | 48 +- .../sources/SourcesGenerationStrategy.java | 2 +- .../pig/impl/sources/SourcesGenerator.java | 11 + .../bacon/pig/impl/utils/CollectionUtils.java | 6 +- .../pig/impl/utils/FileDownloadUtils.java | 18 +- .../pnc/bacon/pig/impl/utils/FileUtils.java | 87 +- .../jboss/pnc/bacon/pig/impl/utils/GAV.java | 30 +- .../pnc/bacon/pig/impl/utils/JarUtils.java | 22 - .../pnc/bacon/pig/impl/utils/LoggerUtils.java | 51 - .../pig/impl/utils/MavenRepositoryUtils.java | 2 +- .../pnc/bacon/pig/impl/utils/OSCheck.java | 58 - .../pig/impl/utils/OSCommandExecutor.java | 12 +- .../bacon/pig/impl/utils/ResourceUtils.java | 24 +- .../pnc/bacon/pig/impl/utils/XmlUtils.java | 8 +- .../pnc/bacon/pig/impl/utils/indy/Indy.java | 37 + .../{javadoc => utils/pom}/Dependency.java | 4 +- .../pom}/DependencyManagement.java | 4 +- .../impl/{javadoc => utils/pom}/Profile.java | 4 +- .../impl/{javadoc => utils/pom}/Profiles.java | 4 +- .../impl/{javadoc => utils/pom}/Project.java | 4 +- pig/src/main/resources/ARTIFACT_LIST.ftl | 1 + .../resources/DUPLICATE_ARTIFACT_LIST.ftl | 1 + pig/src/main/resources/README.ftl | 56 + .../main/resources/REPOSITORY_CONTENTS.ftl | 1 + .../main/resources/REPOSITORY_COORDINATES.ftl | 4 + pig/src/main/resources/generate-nvr-list.sh | 47 + pig/src/main/resources/indy-settings.xml | 56 + pig/src/main/resources/indy-temp-settings.xml | 69 + .../resources/koji-build-finder/config.json | 10 + .../resources/license-generator.properties | 4 + pig/src/main/resources/pom-template.xml | 63 + pig/src/main/resources/release.sh | 38 + pig/src/main/resources/repository-README.md | 4 + .../resources/repository-example-settings.xml | 54 + .../main/resources/rh-license-exceptions.json | 1113 +++++++++++++++++ pig/src/main/resources/settings-template.xml | 57 + 83 files changed, 3136 insertions(+), 714 deletions(-) create mode 100644 config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java rename integration-tests/src/test/resources/config/{pig.yaml => build-config.yaml} (100%) create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/{Config.java => PigConfiguration.java} (92%) create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/JarUtils.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/LoggerUtils.java delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/OSCheck.java create mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/utils/indy/Indy.java rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Dependency.java (96%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/DependencyManagement.java (91%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Profile.java (94%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Profiles.java (89%) rename pig/src/main/java/org/jboss/pnc/bacon/pig/impl/{javadoc => utils/pom}/Project.java (95%) create mode 100644 pig/src/main/resources/ARTIFACT_LIST.ftl create mode 100644 pig/src/main/resources/DUPLICATE_ARTIFACT_LIST.ftl create mode 100644 pig/src/main/resources/README.ftl create mode 100644 pig/src/main/resources/REPOSITORY_CONTENTS.ftl create mode 100644 pig/src/main/resources/REPOSITORY_COORDINATES.ftl create mode 100755 pig/src/main/resources/generate-nvr-list.sh create mode 100644 pig/src/main/resources/indy-settings.xml create mode 100644 pig/src/main/resources/indy-temp-settings.xml create mode 100644 pig/src/main/resources/koji-build-finder/config.json create mode 100644 pig/src/main/resources/license-generator.properties create mode 100644 pig/src/main/resources/pom-template.xml create mode 100644 pig/src/main/resources/release.sh create mode 100644 pig/src/main/resources/repository-README.md create mode 100644 pig/src/main/resources/repository-example-settings.xml create mode 100644 pig/src/main/resources/rh-license-exceptions.json create mode 100644 pig/src/main/resources/settings-template.xml diff --git a/README.md b/README.md index b24847dc2..58feb56a0 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,11 @@ The all-in-one phase that combines all the phases is: The `configure` phase pushes all the settings in the YAML file to PNC, and the `build` phase tells PNC to build everything. ```bash -bacon pig -c ... +bacon pig ... ``` ## Configuration -The application is configured via the `build-config.yaml` file. The file path is passed to the application with `-c` option. +The application is configured via the `build-config.yaml` file. A PiG `build-config.yaml` looks like this: https://github.com/project-ncl/bacon/blob/master/example-pig-config.yaml @@ -109,14 +109,14 @@ TODO: add a command that allows users to specify variables via cli Usage: ```bash -bacon pig configure -c +bacon pig configure ``` ## build Usage: ```bash -bacon pig configure -c +bacon pig configure ``` You can specify if you want temporary builds or not with the `-t` flag. diff --git a/common/src/main/java/org/jboss/pnc/bacon/common/cli/AbstractCommand.java b/common/src/main/java/org/jboss/pnc/bacon/common/cli/AbstractCommand.java index a2ea3ae22..bd1e4b86d 100644 --- a/common/src/main/java/org/jboss/pnc/bacon/common/cli/AbstractCommand.java +++ b/common/src/main/java/org/jboss/pnc/bacon/common/cli/AbstractCommand.java @@ -17,8 +17,8 @@ */ package org.jboss.pnc.bacon.common.cli; -import com.fasterxml.jackson.core.JsonProcessingException; import ch.qos.logback.classic.Level; +import com.fasterxml.jackson.core.JsonProcessingException; import lombok.extern.slf4j.Slf4j; import org.aesh.command.Command; import org.aesh.command.CommandException; @@ -27,10 +27,10 @@ import org.aesh.command.invocation.CommandInvocation; import org.aesh.command.option.Option; import org.aesh.command.shell.Shell; -import org.jboss.pnc.bacon.common.ObjectHelper; import org.jboss.pnc.bacon.common.Constant; -import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.common.ObjectHelper; import org.jboss.pnc.bacon.common.exception.FatalException; +import org.jboss.pnc.bacon.config.Config; import org.jboss.pnc.client.ClientException; /** @@ -206,7 +206,7 @@ public CommandResult executeHelper(CommandInvocation commandInvocation, SubComma } catch (FatalException e) { throw e; } catch (Exception e) { - log.error("Something wrong happened: {}", e.getMessage()); + log.error("Something wrong happened: ", e); // TODO rollback to print message only? log.debug("Stacktrace", e); throw new FatalException(); // Aesh doesnt take care of exit codes, thrown FatalException will be caught in App class and app will exit diff --git a/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java b/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java index ad3c096bc..544bfc250 100644 --- a/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java +++ b/config/src/main/java/org/jboss/pnc/bacon/config/ConfigProfile.java @@ -29,6 +29,7 @@ public class ConfigProfile { private PncConfig pnc; private DaConfig da; private IndyConfig indy; + private PigConfig pig; private KeycloakConfig keycloak; private Map> addOns; diff --git a/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java b/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java new file mode 100644 index 000000000..410512c31 --- /dev/null +++ b/config/src/main/java/org/jboss/pnc/bacon/config/PigConfig.java @@ -0,0 +1,13 @@ +package org.jboss.pnc.bacon.config; + +import lombok.Data; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +@Data +public class PigConfig { + private String kojiHubUrl; + private String licenseServiceUrl; + private String indyUrl; +} diff --git a/integration-tests/src/test/java/org/jboss/pnc/bacon/test/CLIExecutor.java b/integration-tests/src/test/java/org/jboss/pnc/bacon/test/CLIExecutor.java index 8a044f052..4779a86e1 100644 --- a/integration-tests/src/test/java/org/jboss/pnc/bacon/test/CLIExecutor.java +++ b/integration-tests/src/test/java/org/jboss/pnc/bacon/test/CLIExecutor.java @@ -1,5 +1,7 @@ package org.jboss.pnc.bacon.test; +import org.jboss.pnc.bacon.common.Constant; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -12,7 +14,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.jboss.pnc.bacon.common.Constant; /** * @@ -20,7 +21,7 @@ */ public class CLIExecutor { private static final Path BACON_JAR = Paths.get("../cli/target/bacon.jar").toAbsolutePath().normalize(); - public static final String CONFIG_LOCATION = "target/test-config"; + public static final Path CONFIG_LOCATION = Paths.get("target", "test-config"); public ExecutionResult runCommand(String... args) { try { @@ -34,7 +35,7 @@ public ExecutionResult runCommand(String... args) { cmdarray[i + 3] = args[i]; } String[] env = new String[1]; - env[0] = Constant.CONFIG_ENV + "=" + CONFIG_LOCATION; + env[0] = Constant.CONFIG_ENV + "=" + CONFIG_LOCATION.toString(); System.out.println( "Running command: " + Arrays.stream(cmdarray).collect(Collectors.joining("' '", "'", "'"))); diff --git a/integration-tests/src/test/java/org/jboss/pnc/bacon/test/pig/PigTest.java b/integration-tests/src/test/java/org/jboss/pnc/bacon/test/pig/PigTest.java index 9fb78d802..f777deab9 100644 --- a/integration-tests/src/test/java/org/jboss/pnc/bacon/test/pig/PigTest.java +++ b/integration-tests/src/test/java/org/jboss/pnc/bacon/test/pig/PigTest.java @@ -1,16 +1,32 @@ package org.jboss.pnc.bacon.test.pig; -import com.fasterxml.jackson.core.JsonProcessingException; +import org.jboss.pnc.bacon.test.AbstractTest; +import org.jboss.pnc.bacon.test.ExecutionResult; +import org.jboss.pnc.dto.BuildConfiguration; +import org.jboss.pnc.dto.Environment; +import org.jboss.pnc.dto.GroupConfiguration; +import org.jboss.pnc.dto.Product; +import org.jboss.pnc.dto.ProductMilestone; +import org.jboss.pnc.dto.ProductVersion; +import org.jboss.pnc.dto.Project; +import org.jboss.pnc.dto.SCMRepository; +import org.jboss.pnc.dto.response.Page; +import org.jboss.pnc.dto.response.RepositoryCreationResponse; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; + import static org.assertj.core.api.Assertions.assertThat; -import org.jboss.pnc.bacon.test.AbstractTest; import static org.jboss.pnc.bacon.test.CLIExecutor.CONFIG_LOCATION; import static org.jboss.pnc.bacon.test.Endpoints.BUILD_CONFIG; import static org.jboss.pnc.bacon.test.Endpoints.BUILD_CONFIG_DEPENDENCIES; @@ -24,22 +40,6 @@ import static org.jboss.pnc.bacon.test.Endpoints.PROJECT; import static org.jboss.pnc.bacon.test.Endpoints.SCM_REPOSITORY; import static org.jboss.pnc.bacon.test.Endpoints.SCM_REPOSITORY_CREATE; -import org.jboss.pnc.bacon.test.ExecutionResult; -import org.jboss.pnc.dto.BuildConfiguration; -import org.jboss.pnc.dto.Environment; -import org.jboss.pnc.dto.GroupConfiguration; -import org.jboss.pnc.dto.Product; -import org.jboss.pnc.dto.ProductMilestone; -import org.jboss.pnc.dto.ProductVersion; -import org.jboss.pnc.dto.Project; -import org.jboss.pnc.dto.SCMRepository; -import org.jboss.pnc.dto.response.Page; -import org.jboss.pnc.dto.response.RepositoryCreationResponse; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestMethodOrder; /** * @@ -64,9 +64,9 @@ public class PigTest extends AbstractTest { @Test @Order(1) - public void shouldCreateProduct() throws JsonProcessingException, IOException { - final String configFile = CONFIG_LOCATION + "/pig.yaml"; - replaceSuffixInConfigFile(configFile); + public void shouldCreateProduct() throws IOException { + final Path configFile = CONFIG_LOCATION; + replaceSuffixInConfigFile(configFile.resolve("build-config.yaml")); final Product product = Product.builder() .id(UNIVERSAL_ID) @@ -139,12 +139,11 @@ public void shouldCreateProduct() throws JsonProcessingException, IOException { .post(GROUP_CONFIG_BUILD_CONFIGS.apply(UNIVERSAL_ID)) .then() .getEntity(GROUP_CONFIG, groupConfigWithBuildConfig); - ExecutionResult output = executeAndGetResult("pig", "configure", "-c", configFile); + ExecutionResult output = executeAndGetResult("pig", "configure", configFile.toString()); assertThat(output.getOutput()).contains("name: \"Product Foobar " + SUFFIX + "\""); } - private void replaceSuffixInConfigFile(String configFile) throws IOException { - final Path configPath = Paths.get(configFile); + private void replaceSuffixInConfigFile(Path configPath) throws IOException { List fileContent = Files.lines(configPath) .map(l -> l.contains("#!suffix=") ? "#!suffix=" + SUFFIX : l) .collect(Collectors.toList()); diff --git a/integration-tests/src/test/resources/config/pig.yaml b/integration-tests/src/test/resources/config/build-config.yaml similarity index 100% rename from integration-tests/src/test/resources/config/pig.yaml rename to integration-tests/src/test/resources/config/build-config.yaml diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java index 6fbbab242..87a8d2225 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java @@ -22,6 +22,7 @@ import org.aesh.command.CommandResult; import org.aesh.command.GroupCommandDefinition; import org.aesh.command.invocation.CommandInvocation; +import org.aesh.command.option.Argument; import org.aesh.command.option.Option; import org.jboss.pnc.bacon.common.ObjectHelper; import org.jboss.pnc.bacon.common.cli.AbstractCommand; @@ -56,21 +57,15 @@ public class Pig extends AbstractCommand { public static final String TEMP_BUILD_DEFAULT = "false"; public static final String TEMP_BUILD = "tempBuild"; public static final char TEMP_BUILD_SHORT = 't'; - public static final String CONFIG_DESC = "Location of Pig configuration file"; - public static final String CONFIG_DEFAULT = "build-config.yaml"; - public static final char CONFIG_SHORT = 'c'; public static final String REMOVE_M2_DUPLICATES_DESC = "If enabled, only the newest versions of each of the dependencies (groupId:artifactId) " + "are kept in the generated repository zip"; public static final String REMOVE_M2_DUPLICATES = "removeGeneratedM2Dups"; public static final String REMOVE_M2_DUPLICATES_DEFAULT = "false"; public abstract class PigCommand extends AbstractCommand { - @Option( - shortName = CONFIG_SHORT, - overrideRequired = true, - defaultValue = CONFIG_DEFAULT, - description = CONFIG_DESC) - String config; + @Argument(required = true, description = "Directory containing the Pig configuration file") + String configDir; + @Option( shortName = 'o', overrideRequired = false, @@ -83,7 +78,7 @@ public CommandResult execute(CommandInvocation commandInvocation) throws CommandException, InterruptedException { return super.executeHelper(commandInvocation, () -> { - PigContext.get().loadConfig(config); + PigContext.get().loadConfig(configDir); ObjectHelper.print(jsonOutput, doExecute()); }); } @@ -190,7 +185,7 @@ public class Run extends PigCommand { @Override public String doExecute() { - PigContext.get().loadConfig(config); + PigContext.get().loadConfig(configDir); return PigFacade.run( skipRepo, skipPncUpdate, diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java index 104b055a5..68ac628a4 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java @@ -20,7 +20,7 @@ import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; import org.jboss.pnc.bacon.pig.impl.addons.AddOnFactory; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.DocumentGenerator; import org.jboss.pnc.bacon.pig.impl.javadoc.JavadocManager; import org.jboss.pnc.bacon.pig.impl.license.LicenseManager; @@ -190,7 +190,9 @@ public static String run( } public static void generateScripts() { - ScriptGenerator scriptGenerator = new ScriptGenerator(context().getConfig(), context().getDeliverables()); + ScriptGenerator scriptGenerator = new ScriptGenerator( + context().getPigConfiguration(), + context().getDeliverables()); scriptGenerator.generateReleaseScripts( context().getPncImportResult().getMilestone(), context().getRepositoryData().getRepositoryPath(), @@ -202,7 +204,7 @@ public static void generateScripts() { public static void generateDocuments() { DocumentGenerator docGenerator = new DocumentGenerator( - context().getConfig(), + context().getPigConfiguration(), context().getReleasePath(), context().getExtrasPath(), context().getDeliverables()); @@ -212,7 +214,7 @@ public static void generateDocuments() { public static void prepareSharedContentAnalysis() { try { DocumentGenerator docGenerator = new DocumentGenerator( - context().getConfig(), + context().getPigConfiguration(), context().getReleasePath(), context().getExtrasPath(), context().getDeliverables()); @@ -223,12 +225,12 @@ public static void prepareSharedContentAnalysis() { } public static void generateSources() { - Config config = context().getConfig(); + PigConfiguration pigConfiguration = context().getPigConfiguration(); Map builds = context().getBuilds(); RepositoryData repo = context().getRepositoryData(); SourcesGenerator sourcesGenerator = new SourcesGenerator( - config.getFlow().getSourcesGeneration(), - config.getTopLevelDirectoryPrefix() + "src", + pigConfiguration.getFlow().getSourcesGeneration(), + pigConfiguration.getTopLevelDirectoryPrefix() + "src", context().getReleasePath() + context().getDeliverables().getSourceZipName()); sourcesGenerator.generateSources(builds, repo); } @@ -275,7 +277,7 @@ private static RepositoryData parseRepository(File repositoryZipPath) { public static void triggerAddOns() { AddOnFactory .listAddOns( - context().getConfig(), + context().getPigConfiguration(), context().getBuilds(), context().getReleasePath(), context().getExtrasPath(), @@ -288,7 +290,7 @@ public static void triggerAddOns() { public static RepositoryData generateRepo(boolean removeGeneratedM2Dups) { PigContext context = context(); RepoManager repoManager = new RepoManager( - context.getConfig(), + context.getPigConfiguration(), context().getReleasePath(), context().getDeliverables(), context().getBuilds(), @@ -308,15 +310,16 @@ private static Map getBuilds(ImportResult importResult) { public static void generateLicenses() { PigContext context = context(); - Config config = context.getConfig(); + PigConfiguration pigConfiguration = context.getPigConfiguration(); RepositoryData repo = context.getRepositoryData(); Map builds = context.getBuilds(); - new LicenseManager(config, context.getReleasePath(), context.getDeliverables(), builds, repo).prepare(); + new LicenseManager(pigConfiguration, context.getReleasePath(), context.getDeliverables(), builds, repo) + .prepare(); } public static void generateJavadoc() { - Config config = context().getConfig(); + PigConfiguration pigConfiguration = context().getPigConfiguration(); Map builds = context().getBuilds(); - new JavadocManager(config, context().getReleasePath(), context().getDeliverables(), builds).prepare(); + new JavadocManager(pigConfiguration, context().getReleasePath(), context().getDeliverables(), builds).prepare(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java index 69c23399a..2a2d1ddee 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.ImportResult; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -50,7 +51,7 @@ public class PigContext { "pig.context.dir", getProperty("java.io.tmpdir") + File.separator + "pig-context"); - private Config config; // TODO merge config instead of setting it? + private PigConfiguration pigConfiguration; // TODO merge config instead of setting it? private ImportResult pncImportResult; private Map builds; private RepositoryData repositoryData; @@ -60,14 +61,14 @@ public class PigContext { private String releasePath; private String extrasPath; - public void setConfig(Config config) { - this.config = config; + public void setPigConfiguration(PigConfiguration pigConfiguration) { + this.pigConfiguration = pigConfiguration; if (deliverables == null) { String prefix = String.format( "%s-%s.%s", - config.getOutputPrefixes().getReleaseFile(), - config.getVersion(), - config.getMilestone()); + pigConfiguration.getOutputPrefixes().getReleaseFile(), + pigConfiguration.getVersion(), + pigConfiguration.getMilestone()); deliverables = new Deliverables(); @@ -77,13 +78,14 @@ public void setConfig(Config config) { deliverables.setJavadocZipName(prefix + "-javadoc.zip"); deliverables.setNvrListName(prefix + "-nvr-list.txt"); } - configureTargetDirectories(config); + configureTargetDirectories(pigConfiguration); } - private void configureTargetDirectories(Config config) { - String productPrefix = config.getProduct().prefix(); + private void configureTargetDirectories(PigConfiguration pigConfiguration) { + String productPrefix = pigConfiguration.getProduct().prefix(); targetPath = "target"; // TODO: a way to customize it - releasePath = targetPath + "/" + productPrefix + "-" + config.getVersion() + "." + config.getMilestone() + "/"; + releasePath = targetPath + "/" + productPrefix + "-" + pigConfiguration.getVersion() + "." + + pigConfiguration.getMilestone() + "/"; File releaseDirectory = Paths.get(releasePath).toFile(); if (!releaseDirectory.isDirectory()) { @@ -108,16 +110,18 @@ public void storeContext() { } } - public void loadConfig(String config) { - File configFile = new File(config); + public void loadConfig(String configDirStr) { + Path configDir = Paths.get(configDirStr); + File configFile = configDir.resolve("build-config.yaml").toFile(); if (configFile.exists()) { try (FileInputStream configStream = new FileInputStream(configFile)) { - setConfig(Config.load(configStream)); + setPigConfiguration(PigConfiguration.load(configStream)); } catch (IOException e) { throw new RuntimeException("Failed to read config file: " + configFile.getAbsolutePath(), e); } } else { - throw new MisconfigurationException("The provided config file: " + config + " does not exist"); + throw new MisconfigurationException( + "The provided config file: " + configFile.getAbsolutePath() + " does not exist"); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java new file mode 100644 index 000000000..245a6f968 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java @@ -0,0 +1,37 @@ +package org.jboss.pnc.bacon.pig.impl; + +import lombok.Getter; + +import java.util.Properties; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +@Getter +public class PigProperties { + + private boolean temporary; + private boolean removeGeneratedM2Dups; + private boolean skipBranchCheck; + + private PigProperties() { + } + + private static PigProperties instance; + + public static synchronized void init(Properties properties) { + instance = new PigProperties(); + instance.removeGeneratedM2Dups = Boolean.TRUE.toString() + .equalsIgnoreCase(properties.getProperty("removeGeneratedM2Dups", "false")); + instance.skipBranchCheck = Boolean.TRUE.toString() + .equalsIgnoreCase(properties.getProperty("skipBranchCheck", "false")); + instance.temporary = Boolean.TRUE.toString().equalsIgnoreCase(properties.getProperty("temporary", "false")); + } + + public static synchronized PigProperties get() { + if (instance == null) { + init(System.getProperties()); + } + return instance; + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java index d484665e1..281e2bdd2 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOn.java @@ -17,7 +17,7 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import java.util.Map; @@ -27,24 +27,28 @@ * Date: 12/11/17 */ public abstract class AddOn { - protected final Config config; + protected final PigConfiguration pigConfiguration; protected final Map builds; protected final String releasePath; protected final String extrasPath; - protected AddOn(Config config, Map builds, String releasePath, String extrasPath) { - this.config = config; + protected AddOn( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + this.pigConfiguration = pigConfiguration; this.builds = builds; this.releasePath = releasePath; this.extrasPath = extrasPath; } public boolean shouldRun() { - return config.getAddons().keySet().contains(getName()); + return pigConfiguration.getAddons().containsKey(getName()); } - public Map getConfig() { - return config.getAddons().get(getName()); + public Map getPigConfiguration() { + return pigConfiguration.getAddons().get(getName()); } protected abstract String getName(); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java index d31713394..72659088f 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/AddOnFactory.java @@ -17,11 +17,12 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; +import org.jboss.pnc.bacon.pig.impl.addons.microprofile.MicroProfileSmallRyeCommunityDepAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.quarkus.QuarkusCommunityDepAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.runtime.RuntimeDependenciesAnalyzer; import org.jboss.pnc.bacon.pig.impl.addons.spring.BomVerifierAddon; import org.jboss.pnc.bacon.pig.impl.addons.vertx.NotYetAlignedFromDependencyTree; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -35,23 +36,25 @@ */ public class AddOnFactory { - private AddOnFactory() { - } - public static List listAddOns( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath, Deliverables deliverables) { ArrayList resultList = new ArrayList<>(); - resultList.add(new RuntimeDependenciesAnalyzer(config, builds, releasePath, extrasPath)); - resultList.add(new ExtraDeliverableDownloader(config, builds, releasePath, extrasPath)); - resultList.add(new BomVerifierAddon(config, builds, releasePath, extrasPath)); - resultList.add(new NotYetAlignedFromDependencyTree(config, builds, releasePath, extrasPath)); - resultList.add(new QuarkusCommunityDepAnalyzer(config, builds, releasePath, extrasPath, deliverables)); + resultList.add(new RuntimeDependenciesAnalyzer(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new ExtraDeliverableDownloader(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new BomVerifierAddon(pigConfiguration, builds, releasePath, extrasPath)); + resultList.add(new NotYetAlignedFromDependencyTree(pigConfiguration, builds, releasePath, extrasPath)); + resultList + .add(new QuarkusCommunityDepAnalyzer(pigConfiguration, builds, releasePath, extrasPath, deliverables)); + resultList.add(new MicroProfileSmallRyeCommunityDepAnalyzer(pigConfiguration, builds, releasePath, extrasPath)); return resultList; } + + private AddOnFactory() { + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java index 63fa0efa6..07f105f73 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/ExtraDeliverableDownloader.java @@ -17,7 +17,7 @@ */ package org.jboss.pnc.bacon.pig.impl.addons; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -33,11 +33,11 @@ public class ExtraDeliverableDownloader extends AddOn { protected ExtraDeliverableDownloader( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -45,10 +45,11 @@ protected String getName() { return "extraDeliverablesDownloader"; } + @SuppressWarnings("unchecked") @Override public void trigger() { - // noinspection unchecked - getConfig().forEach((key, artifacts) -> downloadArtifacts(key, (List>) artifacts)); + getPigConfiguration() + .forEach((key, artifacts) -> downloadArtifacts(key, (List>) artifacts)); } private void downloadArtifacts(String buildName, List> artifacts) { @@ -66,9 +67,9 @@ private void downloadArtifact(PncBuild build, String pattern, String suffix) { private String constructFileName(String suffix) { return String.format( "%s-%s.%s-%s", - config.getOutputPrefixes().getReleaseFile(), - config.getVersion(), - config.getMilestone(), + pigConfiguration.getOutputPrefixes().getReleaseFile(), + pigConfiguration.getVersion(), + pigConfiguration.getMilestone(), suffix); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java new file mode 100644 index 000000000..b7d1a2e22 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/BuildLogWithDependencyTrees.java @@ -0,0 +1,88 @@ +package org.jboss.pnc.bacon.pig.impl.addons.microprofile; + +import com.google.common.collect.Sets; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class BuildLogWithDependencyTrees { + private static final Logger log = LoggerFactory.getLogger(BuildLogWithDependencyTrees.class); + + private static final Pattern dependencyTreeStart = Pattern + .compile("^\\[INFO] --- maven-dependency-plugin:.*?:tree .*? @ (.*?) ---$"); + private static final Pattern dependencyTreeEnd = Pattern.compile("^\\[INFO][\\s-]*$"); + private static final Set relevantScopes = Sets.newHashSet("compile", "runtime"); + + public final Map> communityGavsForModules; + + public BuildLogWithDependencyTrees(List buildLogLines) { + communityGavsForModules = parseDependencyTreeInvocations(buildLogLines); + } + + private static Map> parseDependencyTreeInvocations(List buildLogLines) { + Map> result = new HashMap<>(); + + String currentModuleName = null; + List currentModuleDependencyTree = null; + + for (String line : buildLogLines) { + Matcher matcher = dependencyTreeStart.matcher(line); + if (matcher.matches()) { + currentModuleName = matcher.group(1); + currentModuleDependencyTree = new ArrayList<>(); + continue; + } + + if (dependencyTreeEnd.matcher(line).matches() && currentModuleDependencyTree != null) { + result.put(currentModuleName, communityGavsInDepTree(currentModuleDependencyTree)); + currentModuleName = null; + currentModuleDependencyTree = null; + continue; + } + + if (currentModuleDependencyTree != null) { + currentModuleDependencyTree.add(line); + } + } + + return result; + } + + private static Set communityGavsInDepTree(List depTreeOutput) { + return depTreeOutput.stream() + .filter(l -> l.startsWith("[INFO] ")) + .map(BuildLogWithDependencyTrees::parseLineToGav) + .filter(Objects::nonNull) + .filter(GAV::isCommunity) + .collect(Collectors.toSet()); + } + + private static GAV parseLineToGav(String mvnDepTreeLine) { + String gavString = mvnDepTreeLine.replaceFirst("\\[INFO] [+|\\\\\\-\\s]+", ""); + String[] splitGav = gavString.split(":"); + if (splitGav.length < 5 || !relevantScopes.contains(splitGav[splitGav.length - 1])) { + return null; + } + switch (splitGav.length) { + case 5: + return new GAV(splitGav[0], splitGav[1], splitGav[3], splitGav[2]); + case 6: + return new GAV(splitGav[0], splitGav[1], splitGav[4], splitGav[2], splitGav[3]); + default: + log.warn( + "suspicious line in the dependency tree '{}', assuming it's not a dependency and skipping", + gavString); + return null; + } + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java new file mode 100644 index 000000000..607094f3c --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/microprofile/MicroProfileSmallRyeCommunityDepAnalyzer.java @@ -0,0 +1,63 @@ +package org.jboss.pnc.bacon.pig.impl.addons.microprofile; + +import org.jboss.pnc.bacon.pig.impl.addons.AddOn; +import org.jboss.pnc.bacon.pig.impl.addons.runtime.CommunityDepAnalyzer; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Performs community dependency analysis on all builds (in "our" build group, that is). Assumes that each build + * includes {@code mvn dependency:tree} in its build script. Community dependencies from modules whose names contain + * {@code test} or {@code tck} are ignored. Only dependencies in the {@code compile} and {@code runtime} scopes are + * included. + */ +public class MicroProfileSmallRyeCommunityDepAnalyzer extends AddOn { + private static final Logger log = LoggerFactory.getLogger(MicroProfileSmallRyeCommunityDepAnalyzer.class); + + public static final String NAME = "microProfileSmallRyeCommunityDepAnalyzer"; + + public MicroProfileSmallRyeCommunityDepAnalyzer( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + super(pigConfiguration, builds, releasePath, extrasPath); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + public void trigger() { + log.info("Running MicroProfileSmallRyeCommunityDepAnalyzer"); + + Set allGavs = new HashSet<>(); + for (PncBuild build : builds.values()) { + Set gavs = new BuildLogWithDependencyTrees(build.getBuildLog()).communityGavsForModules.entrySet() + .stream() + .filter(e -> !e.getKey().contains("test") && !e.getKey().contains("tck")) + .flatMap(e -> e.getValue().stream()) + .collect(Collectors.toSet()); + + Path targetPath = Paths.get(extrasPath, "community-dependencies-" + build.getName() + ".csv"); + new CommunityDepAnalyzer(gavs).generateAnalysis(targetPath.toAbsolutePath().toString()); + + allGavs.addAll(gavs); + } + + Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); + new CommunityDepAnalyzer(allGavs).generateAnalysis(targetPath.toAbsolutePath().toString()); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java index 2e3d87b1c..d85f4b8ee 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/quarkus/QuarkusCommunityDepAnalyzer.java @@ -5,17 +5,24 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.collect.Sets; import lombok.SneakyThrows; - +import org.apache.maven.model.Dependency; +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; import org.jboss.pnc.bacon.pig.impl.addons.runtime.CommunityDepAnalyzer; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; -import org.jboss.pnc.bacon.pig.impl.utils.*; - +import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.jboss.pnc.bacon.pig.impl.utils.MavenRepositoryUtils; +import org.jboss.pnc.bacon.pig.impl.utils.OSCommandExecutor; +import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -30,6 +37,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -61,12 +69,12 @@ public class QuarkusCommunityDepAnalyzer extends AddOn { } public QuarkusCommunityDepAnalyzer( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath, Deliverables deliverables) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); this.deliverables = deliverables; } @@ -83,7 +91,7 @@ public void trigger() { Path repoZipPath = Paths.get(releasePath + deliverables.getRepositoryZipName()); unpackRepository(repoZipPath); - String additionalRepository = (String) getConfig().get("additionalRepository"); + String additionalRepository = (String) getPigConfiguration().get("additionalRepository"); String settingsSelector = ""; if (additionalRepository != null) { try { @@ -114,6 +122,15 @@ public void trigger() { Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); depAnalyzer.generateAnalysis(targetPath.toAbsolutePath().toString()); + + Set problematicDeps = gatherProblematicDeps(); + Path problematicDepsOut = Paths.get(extrasPath, "nonexistent-redhat-deps.txt"); + + try (BufferedWriter writer = Files.newBufferedWriter(problematicDepsOut)) { + writer.write(String.join("\n", problematicDeps)); + } catch (IOException e) { + throw new RuntimeException("Failed to write problematic dependencies to the output file", e); + } } private Path buildReactiveProject(String settingsSelector) { @@ -130,7 +147,7 @@ private Path buildNonReactiveProject(String settingsSelector) { private Collection skippedExtensions() { // noinspection unchecked - Collection skipped = (Collection) getConfig().get("skippedExtensions"); + Collection skipped = (Collection) getPigConfiguration().get("skippedExtensions"); return skipped == null ? Collections.emptyList() : skipped; } @@ -156,8 +173,8 @@ private void unpackRepository(Path repoZipPath) { @SneakyThrows private Set listDependencies(Path projectPath, Path depThreeOut, String settingsSelector) { - List result = OSCommandExecutor - .runCommandIn("mvn dependency:tree" + repoDefinition + settingsSelector, projectPath); + List result = OSCommandExecutor // mstodo!!! + .runCommandIn("mvn dependency:tree " + repoDefinition + settingsSelector, projectPath); Files.write(depThreeOut, result); return depTreeToNonRedhatGAVs(result); @@ -279,4 +296,72 @@ private List findAllByExtension(String extension) { } return visitor.getFilePaths(); } + + private Set gatherProblematicDeps() { + Set problemmaticDeps = new TreeSet<>(); + problemmaticDeps.addAll(checkBomContents(".*/io/quarkus/quarkus-bom/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/io/quarkus/quarkus-bom-deployment/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/com/redhat/quarkus/quarkus-product-bom/.*\\.pom")); + problemmaticDeps.addAll(checkBomContents(".*/com/redhat/quarkus/quarkus-product-bom-deployment/.*\\.pom")); + return problemmaticDeps; + } + + private Collection checkBomContents(String bomLocator) { + String quarkusRuntimeBom = repoZipContents.stream() + // todo file separator for Wi***ws may be different + .filter(file -> file.matches(bomLocator)) + .findAny() + .get(); + return checkReferencesInRepo(quarkusRuntimeBom); + } + + // mstodo store the result properly + // mstodo add it add on trigger + private Collection checkReferencesInRepo(String quarkusRuntimeBom) { + try { + String str = "/maven-repository/"; + int repoDirIdx = quarkusRuntimeBom.indexOf(str); + quarkusRuntimeBom = quarkusRuntimeBom.substring(repoDirIdx + str.length()); + Path bomFile = repoPath.resolve(quarkusRuntimeBom).toAbsolutePath(); + Model model = new MavenXpp3Reader().read(Files.newInputStream(bomFile)); + + List dependencies = model.getDependencyManagement().getDependencies(); + + return dependencies.stream() + .filter(dep -> isRHAndMissing(dep, model)) + .map( + dep -> String.format( + "'%s:%s:%s',", + dep.getGroupId(), + dep.getArtifactId(), + deVar(model, dep.getVersion()))) + .collect(Collectors.toSet()); + } catch (XmlPullParserException | IOException e) { + log.error("Parsing error when generating quarkus artifact references", e); + return Collections.emptySet(); + } + } + + private boolean isRHAndMissing(Dependency dependency, Model model) { + String version = dependency.getVersion(); + version = deVar(model, version); + if (!version.contains("redhat")) { + return false; + } + GAV gav = new GAV( + dependency.getGroupId(), + dependency.getArtifactId(), + version, + dependency.getType() != null && dependency.getType().equals("jar") ? dependency.getType() : "jar"); + Path filePath = repoPath.resolve(gav.toVersionPath()).resolve(gav.toFileName()); + return !filePath.toFile().exists(); + } + + private String deVar(Model model, String version) { + if (version.startsWith("$")) { + version = version.substring(2, version.length() - 1); + version = model.getProperties().getProperty(version); + } + return version; + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java index 10ad48c4a..137f8f406 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/CommunityDepAnalyzer.java @@ -18,6 +18,7 @@ package org.jboss.pnc.bacon.pig.impl.addons.runtime; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CommunityDependency; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CsvExportable; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.DADao; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.slf4j.Logger; @@ -27,6 +28,7 @@ import java.io.FileWriter; import java.util.Collection; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -39,40 +41,34 @@ public class CommunityDepAnalyzer { private final DADao daDao; private final List dependencies; - private final List downloadedForSwarm; + private final Function, List> enricher; + private boolean skipDa = false; - public CommunityDepAnalyzer(List dependencyLines, List swarmLog) { + public CommunityDepAnalyzer( + List dependencyLines, + Function, List> enricher) { daDao = DADao.getInstance(); - downloadedForSwarm = swarmLog.stream() - .filter(line -> line.startsWith("Downloaded")) - .filter(line -> line.contains(".jar")) - // lines are of the form: Downloaded: http://... (some add. info) - .map(l -> l.split("\\s+")[1]) - .sorted() - .collect(Collectors.toList()); dependencies = dependencyLines.stream().map(CommunityDependency::new).collect(Collectors.toList()); + this.enricher = enricher; } public CommunityDepAnalyzer(Collection gavs) { dependencies = gavs.stream().map(CommunityDependency::new).collect(Collectors.toList()); daDao = null; - downloadedForSwarm = null; + enricher = null; } public File generateAnalysis(String path) { try { - if (!skipDa) { - analyzeDAResults(); - } - analyzeSwarmBuildLog(); log.info("generating analysis to {} ...", path); + List csvContents = analyze(); File csvFile = new File(path); try (FileWriter writer = new FileWriter(csvFile)) { writer.append( - "Community dependencies in Swarm;;Productized counterpart;Other productized versions;Used in the Swarm build\n"); - dependencies.forEach(d -> d.appendToCsv(writer)); + "Community dependencies;;Productized counterpart;Other productized versions;Additional info\n"); + csvContents.forEach(d -> d.appendToCsv(writer)); } log.info("DONE"); @@ -82,22 +78,20 @@ public File generateAnalysis(String path) { } } - protected List analyzeDAResults() { - dependencies.parallelStream().forEach(daDao::fillDaData); - return dependencies; + protected List analyze() { + if (!skipDa) { + analyzeDAResults(); + } + return productSpecificAnalysis(); } - protected List analyzeSwarmBuildLog() { - dependencies.parallelStream().forEach(this::addSwarmBuildDependencies); - return dependencies; + private List productSpecificAnalysis() { + return enricher != null ? enricher.apply(dependencies) : dependencies; } - private void addSwarmBuildDependencies(CommunityDependency communityDependency) { - List swarmBuildDownloads = downloadedForSwarm.stream() - .filter(d -> d.contains(communityDependency.toPathSubstring())) - .map(l -> l.substring(l.lastIndexOf("/") + 1)) - .collect(Collectors.toList()); - communityDependency.setUsedForSwarm(swarmBuildDownloads); + protected List analyzeDAResults() { + dependencies.parallelStream().forEach(daDao::fillDaData); + return dependencies; } public void skipDa(boolean skip) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java deleted file mode 100644 index f8335265b..000000000 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RepoBuildLogProcessor.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * Copyright 2017 Red Hat, Inc., and individual contributors - * as indicated by the @author tags. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jboss.pnc.bacon.pig.impl.addons.runtime; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import static org.apache.commons.lang3.StringUtils.join; - -/** - * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
- * Date: 6/5/17 - */ -public class RepoBuildLogProcessor { - private static final String START_TAG = "swarm_repository_listing_begin"; - private static final String END_TAG = "swarm_repository_listing_end"; - - public static List getList(String header, List buildLog) { - Iterator logIterator = buildLog.iterator(); - while (logIterator.hasNext()) { - if (header.equals(logIterator.next())) { - break; - } - } - if (logIterator.hasNext()) { - if (START_TAG.equals(logIterator.next()) && logIterator.hasNext()) { - return takeUntilTheEnd(logIterator); - } else { - throw new IllegalArgumentException( - "Malformed repo log listing for header: " + header + " in the log: " + join(buildLog, "\n")); - } - } else { - throw new IllegalArgumentException("Unable to find " + header + " in the log: " + join(buildLog, "\n")); - } - } - - private static List takeUntilTheEnd(Iterator logIterator) { - List resultList = new ArrayList<>(); - while (logIterator.hasNext()) { - String nextLine = logIterator.next(); - if (END_TAG.equals(nextLine)) { - break; - } - resultList.add(nextLine); - } - return resultList; - } - - private RepoBuildLogProcessor() { - } -} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java index 28384e8cc..4ae787c59 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/RuntimeDependenciesAnalyzer.java @@ -19,7 +19,7 @@ import org.apache.commons.io.IOUtils; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.slf4j.Logger; @@ -42,11 +42,11 @@ public class RuntimeDependenciesAnalyzer extends AddOn { private static final Logger log = LoggerFactory.getLogger(RuntimeDependenciesAnalyzer.class); public RuntimeDependenciesAnalyzer( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -59,9 +59,9 @@ public void trigger() { log.info("Running RuntimeDependenciesAnalyzer"); File workDir = FileUtils.mkTempDir("runtimeDepAnalyzer"); - String buildName = (String) getConfig().get("downloadFrom"); - String regex = (String) getConfig().get("matching"); - String referenceBuildName = (String) getConfig().get("referenceBuild"); + String buildName = (String) getPigConfiguration().get("downloadFrom"); + String regex = (String) getPigConfiguration().get("matching"); + String referenceBuildName = (String) getPigConfiguration().get("referenceBuild"); File dependencyListPath = new File(workDir, "runtime-dependency-list.txt"); builds.get(buildName).downloadArtifact(regex, dependencyListPath); @@ -82,8 +82,10 @@ public void trigger() { Path targetPath = Paths.get(extrasPath, "community-dependencies.csv"); - CommunityDepAnalyzer analyzer = new CommunityDepAnalyzer(communityDependencies, buildLog); - analyzer.skipDa(false); // TODO: + CommunityDepAnalyzer analyzer = new CommunityDepAnalyzer( + communityDependencies, + new ThorntailDepAnalyzer(buildLog)); + analyzer.skipDa(false); analyzer.generateAnalysis(targetPath.toAbsolutePath().toString()); log.info("Done"); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java new file mode 100644 index 000000000..2793d2076 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/runtime/ThorntailDepAnalyzer.java @@ -0,0 +1,42 @@ +package org.jboss.pnc.bacon.pig.impl.addons.runtime; + +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CommunityDependency; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.CsvExportable; +import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da.ThorntailCommunityDependency; + +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +public class ThorntailDepAnalyzer implements Function, List> { + private final List downloadedForSwarm; + + public ThorntailDepAnalyzer(List swarmLog) { + downloadedForSwarm = swarmLog.stream() + .filter(line -> line.startsWith("Downloaded")) + .filter(line -> line.contains(".jar")) + // lines are of the form: Downloaded: http://... (some add. info) + .map(l -> l.split("\\s+")[1]) + .sorted() + .collect(Collectors.toList()); + } + + @Override + public List apply(List communityDependencies) { + return communityDependencies.parallelStream() + .map(this::toThorntailCommunityDependency) + .collect(Collectors.toList()); + } + + private ThorntailCommunityDependency toThorntailCommunityDependency(CommunityDependency communityDependency) { + List swarmBuildDownloads = downloadedForSwarm.stream() + .filter(d -> d.contains(communityDependency.toPathSubstring())) + .map(l -> l.substring(l.lastIndexOf("/") + 1)) + .collect(Collectors.toList()); + return new ThorntailCommunityDependency(communityDependency, swarmBuildDownloads); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java index b4f527e50..cf5624aed 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/spring/BomVerifierAddon.java @@ -17,7 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.MRRCSearcher; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.GAV; @@ -47,8 +47,12 @@ public class BomVerifierAddon extends AddOn { private final MRRCSearcher mrrcSearcher = MRRCSearcher.getInstance(); private List unreleasedWhitelist; - public BomVerifierAddon(Config config, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + public BomVerifierAddon( + PigConfiguration pigConfiguration, + Map builds, + String releasePath, + String extrasPath) { + super(pigConfiguration, builds, releasePath, extrasPath); } @Override @@ -75,9 +79,10 @@ private List readUnreleasedWhitelist() { }).collect(Collectors.toList()); } + @SuppressWarnings("unchecked") private List getListFromConfig() { try { - Map config = getConfig(); + Map config = getPigConfiguration(); if (config != null) { List result = (List) config.get("allowUnreleased"); return result == null ? emptyList() : result; @@ -111,10 +116,11 @@ private boolean unreleased(GAV gav) { } protected Stream getDependencyGavs() { - PncBuild build = builds.get(config.getFlow().getRepositoryGeneration().getSourceBuild()); + PncBuild build = builds.get(pigConfiguration.getFlow().getRepositoryGeneration().getSourceBuild()); File bom = new File("bom"); - build.findArtifactByFileName(config.getFlow().getRepositoryGeneration().getSourceArtifact()).downloadTo(bom); + build.findArtifactByFileName(pigConfiguration.getFlow().getRepositoryGeneration().getSourceArtifact()) + .downloadTo(bom); List nodeList = listNodes(bom, "//dependencies/dependency"); Map properties = XmlUtils.getProperties(bom); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java index 8eda8c30c..9688e197a 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/addons/vertx/NotYetAlignedFromDependencyTree.java @@ -16,7 +16,7 @@ package org.jboss.pnc.bacon.pig.impl.addons.vertx; import org.jboss.pnc.bacon.pig.impl.addons.AddOn; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,11 +39,11 @@ public class NotYetAlignedFromDependencyTree extends AddOn { private static final Logger log = LoggerFactory.getLogger(NotYetAlignedFromDependencyTree.class); public NotYetAlignedFromDependencyTree( - Config config, + PigConfiguration pigConfiguration, Map builds, String releasePath, String extrasPath) { - super(config, builds, releasePath, extrasPath); + super(pigConfiguration, builds, releasePath, extrasPath); } @Override diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java index cb71369e6..e224b928a 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/common/DeliverableManager.java @@ -18,8 +18,8 @@ package org.jboss.pnc.bacon.pig.impl.common; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; @@ -56,18 +56,18 @@ public abstract class DeliverableManager, private static final Logger log = LoggerFactory.getLogger(DeliverableManager.class); - protected final Config config; + protected final PigConfiguration pigConfiguration; protected final String releasePath; protected final Deliverables deliverables; protected final Map builds; protected final File workDir; protected DeliverableManager( - Config config, + PigConfiguration pigConfiguration, String releasePath, Deliverables deliverables, Map builds) { - this.config = config; + this.pigConfiguration = pigConfiguration; this.releasePath = releasePath; this.deliverables = deliverables; this.builds = builds; diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java index 359e2417c..99480d4bc 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.pnc.GitRepoInspector; import org.jboss.pnc.dto.BuildConfiguration; import org.jboss.pnc.dto.SCMRepository; @@ -39,6 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -60,13 +62,12 @@ public class BuildConfig { private String name; private String project; private String buildScript; - private String buildType; - private String scmUrl; private String externalScmUrl; private String scmRevision; private String description; private String environmentId; + private List dependencies = new ArrayList<>(); /** * build pod memory in GB @@ -76,11 +77,11 @@ public class BuildConfig { private Set customPmeParameters = new TreeSet<>(); private Set extraRepositories = new TreeSet<>(); + private Boolean branchModified; + private String buildType; private String executionRoot; - private List dependencies = new ArrayList<>(); private Set alignmentParameters = new TreeSet<>(); - private Boolean branchModified; /** * Set the defaults of buildConfig if not explicitly specified @@ -89,6 +90,7 @@ public class BuildConfig { * * @param defaults */ + @SuppressWarnings("rawtypes") public void setDefaults(BuildConfig defaults) { try { for (Field f : BuildConfig.class.getDeclaredFields()) { @@ -121,6 +123,20 @@ public void sanitizebuildScript() { buildScript = buildScript.trim(); } + @JsonIgnore + public synchronized boolean isBranchModified(BuildConfiguration oldVersion) { + if (PigProperties.get().isSkipBranchCheck()) { + return false; + } + if (branchModified == null) { + branchModified = GitRepoInspector.isModifiedBranch( + oldVersion.getId(), + oldVersion.getScmRepository().getInternalUrl(), + getScmRevision()); + } + return branchModified; + } + @JsonIgnore public boolean isTheSameAs(BuildConfiguration old) { return old != null && StringUtils.equals(name, old.getName()) @@ -138,16 +154,6 @@ private Set getAlignmentParameters(BuildConfiguration old) { return Arrays.stream(parametersAsString.split(",")).collect(Collectors.toSet()); } - private synchronized boolean isBranchModified(BuildConfiguration oldVersion) { - if (branchModified == null) { - branchModified = GitRepoInspector.isModifiedBranch( - oldVersion.getId(), - oldVersion.getScmRepository().getInternalUrl(), - getScmRevision()); - } - return branchModified; - } - private boolean urlsEqual(SCMRepository repo) { return StringUtils.equals(externalScmUrl, repo.getExternalUrl()) || StringUtils.equals(scmUrl, repo.getInternalUrl()); @@ -162,13 +168,13 @@ public Map getGenericParameters(BuildConfiguration oldConfig, bo } Map result = new HashMap<>(); - String oldForceValue = oldConfig == null ? "" : oldConfig.getParameters().getOrDefault(BUILD_FORCE, ""); - String forceValue = forceRebuild ? randomAlphabetic(5) : oldForceValue; + Optional oldForceValue = oldConfig == null ? Optional.empty() + : Optional.ofNullable(oldConfig.getParameters().get(BUILD_FORCE)); + Optional forceValue = forceRebuild ? Optional.of(randomAlphabetic(5)) : oldForceValue; + forceValue.ifPresent(val -> result.put(BUILD_FORCE, val)); String dependencyExclusions = String.join(" ", alignmentParameters); - result.put("ALIGNMENT_PARAMETERS", dependencyExclusions); - result.put(BUILD_FORCE, forceValue); if (buildPodMemory != null) { result.put("BUILDER_POD_MEMORY", buildPodMemory.toString()); @@ -225,6 +231,9 @@ public boolean matchesRepository(SCMRepository repository) { } private boolean areSameRepoUrls(String scmUrl1, String scmUrl2) { + if (scmUrl1 == null ^ scmUrl2 == null) { + throw new RuntimeException("trying to compare null and non-null scm url: " + scmUrl1 + ", " + scmUrl2); + } String normalizedUrl1 = normalize(scmUrl1); String normalizedUrl2 = normalize(scmUrl2); return normalizedUrl1.equals(normalizedUrl2); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java similarity index 92% rename from pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java rename to pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java index b041a82c5..da548baca 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/Config.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/PigConfiguration.java @@ -52,8 +52,8 @@ * Date: 11/28/17 */ @Data -public class Config { - private static final Logger log = LoggerFactory.getLogger(Config.class); +public class PigConfiguration { + private static final Logger log = LoggerFactory.getLogger(PigConfiguration.class); private @NotBlank ProductConfig product; private @NotBlank String version; @@ -155,7 +155,7 @@ public static InputStream preProcess(InputStream buildConfig, String buildVarsOv } Map variables = readVariables(contents, buildVarsOverrides); - Integer passes = 0; + int passes = 0; // We also have to take into account variable used inside variables, just keep going over until // they have all expanded, maxTries will be hit if one is left over after that many List matches; @@ -187,7 +187,7 @@ public static InputStream preProcess(InputStream buildConfig, String buildVarsOv return stream; } - public static Config load(File buildConfigFile, String buildVarsOverrides) { + public static PigConfiguration load(File buildConfigFile, String buildVarsOverrides) { try (InputStream configStream = new FileInputStream(buildConfigFile)) { return load(configStream, buildVarsOverrides); } catch (IOException e) { @@ -195,21 +195,21 @@ public static Config load(File buildConfigFile, String buildVarsOverrides) { } } - public static Config load(InputStream configStream) { + public static PigConfiguration load(InputStream configStream) { return load(configStream, ""); } - public static Config load(InputStream configStream, String buildVarsOverrides) { + public static PigConfiguration load(InputStream configStream, String buildVarsOverrides) { if (buildVarsOverrides == null) { buildVarsOverrides = ""; } - Yaml yaml = new Yaml(new Constructor(Config.class)); + Yaml yaml = new Yaml(new Constructor(PigConfiguration.class)); try (InputStream in = preProcess(configStream, buildVarsOverrides)) { - Config config = (Config) yaml.load(in); - config.init(); - return config; + PigConfiguration pigConfiguration = (PigConfiguration) yaml.load(in); + pigConfiguration.init(); + return pigConfiguration; } catch (IOException e) { throw new RuntimeException("Unable to load config file", e); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java index 827e0b217..482fc4d63 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationData.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; /** * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
@@ -34,7 +36,16 @@ public class RepoGenerationData extends GenerationData { * list of groupId:artifactId:packaging:version with exact version */ private List externalAdditionalArtifacts = new ArrayList<>(); + private List excludeArtifacts = new ArrayList<>(); + private List sourceBuilds = new ArrayList<>(); private String bomGroupId; private String bomArtifactId; - private Boolean includeJavadoc; + private boolean includeJavadoc; + private boolean includeLicenses; + private boolean includeMavenMetadata; + private String buildScript; + private Set ignored; + private String additionalRepo; + private List> stages; + private Map parameters; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java index 29cf5a9bd..c9d156e96 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/RepoGenerationStrategy.java @@ -22,5 +22,5 @@ * Date: 5/23/18 */ public enum RepoGenerationStrategy { - GENERATE, DOWNLOAD, IGNORE, PACK_ALL + GENERATE, DOWNLOAD, BUILD_CONFIGS, BUILD_GROUP, MILESTONE, IGNORE, PACK_ALL // DEPRECATED replaced by BUILD_CONFIGS } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java index 8093eef6c..fc2f27dba 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DataRoot.java @@ -21,7 +21,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import java.util.Collection; @@ -34,9 +34,10 @@ @AllArgsConstructor @NoArgsConstructor public class DataRoot { - private Config config; + private PigConfiguration pigConfiguration; private Deliverables deliverables; private String duplicates; private String repositoryContents; private Collection builds; + private String pncUrl; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java index e625a0dea..978063877 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/DocumentGenerator.java @@ -17,7 +17,8 @@ */ package org.jboss.pnc.bacon.pig.impl.documents; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.SharedContentReportGenerator; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -44,13 +45,17 @@ public class DocumentGenerator { private final String extrasPath; private final String releasePath; - private final Config config; + private final PigConfiguration pigConfiguration; private final Deliverables deliverables; - public DocumentGenerator(Config config, String releasePath, String extrasPath, Deliverables deliverables) { + public DocumentGenerator( + PigConfiguration pigConfiguration, + String releasePath, + String extrasPath, + Deliverables deliverables) { this.releasePath = releasePath; this.extrasPath = extrasPath; - this.config = config; + this.pigConfiguration = pigConfiguration; this.deliverables = deliverables; } @@ -67,7 +72,13 @@ public void generateDocuments(Map builds, RepositoryData repo) .sorted() .collect(Collectors.joining("\n")); - DataRoot templateData = new DataRoot(config, deliverables, duplicates, repositoryContents, builds.values()); + DataRoot templateData = new DataRoot( + pigConfiguration, + deliverables, + duplicates, + repositoryContents, + builds.values(), + Config.instance().getActiveProfile().getPnc().getUrl()); FileGenerator generator = new FileGenerator(Optional.empty()); log.debug("Generating documents with data: {}", templateData); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java index a4a939aaf..54bbd9afc 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/BrewSearcher.java @@ -19,8 +19,6 @@ package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import com.redhat.red.build.finder.BuildConfig; import com.redhat.red.build.finder.BuildFinder; import com.redhat.red.build.finder.DistributionAnalyzer; @@ -30,12 +28,13 @@ import com.redhat.red.build.koji.model.json.util.KojiObjectMapper; import com.redhat.red.build.koji.model.xmlrpc.KojiArchiveInfo; import com.redhat.red.build.koji.model.xmlrpc.KojiTagInfo; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; @@ -45,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Properties; import java.util.stream.Collectors; /** @@ -60,7 +60,7 @@ public class BrewSearcher { private static final String KOJI_BUILD_FINDER_CONFIG_PROP = "koji.build.finder.config"; - private static final String KOJI_BUILD_FINDER_CONFIG_RES = "koji-build-finder/config.json"; + private static final String KOJI_BUILD_FINDER_CONFIG_TEMPLATE = "koji-build-finder/config.json"; public static BuildConfig getKojiBuildFinderConfigFromFile(final File file) { try { @@ -78,14 +78,12 @@ public static BuildConfig getKojiBuildFinderConfigFromFile(final String filename } public static BuildConfig getKojiBuildFinderConfigFromResource(final String resourceName) { - final URL url = Resources.getResource(resourceName); + Properties props = new Properties(); + props.setProperty("KOJI_URL", Config.instance().getActiveProfile().getPig().getKojiHubUrl()); - try { - final String json = Resources.toString(url, Charsets.UTF_8); - return getKojiBuildFinderConfigFromJson(json); - } catch (IOException e) { - throw new IllegalStateException("Failed read Koji Build Finder configuration from resource: " + url, e); - } + String json = ResourceUtils.extractToStringWithFiltering(KOJI_BUILD_FINDER_CONFIG_TEMPLATE, props); + + return getKojiBuildFinderConfigFromJson(json); } public static BuildConfig getKojiBuildFinderConfigFromJson(final String json) { @@ -112,7 +110,7 @@ public static BuildConfig getKojiBuildFinderConfig() { return getKojiBuildFinderConfigFromFile(envFilename); } - return getKojiBuildFinderConfigFromResource(KOJI_BUILD_FINDER_CONFIG_RES); + return getKojiBuildFinderConfigFromResource(KOJI_BUILD_FINDER_CONFIG_TEMPLATE); } public static void fillBrewData(SharedContentReportRow row) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java index 82629b03e..c8c32e6bd 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/SharedContentReportGenerator.java @@ -40,7 +40,8 @@ public class SharedContentReportGenerator { private static final Logger log = LoggerFactory.getLogger(SharedContentReportGenerator.class); - private static final String[] IGNORED = { "md5", "sha1", "-sources", "example-settings.xml", "README" }; + private static final String[] IGNORED = { "md5", "sha1", "-sources", "example-settings.xml", "README", + "maven-metadata.xml" }; private final Collection repositoryFiles; private final DASearcher daSearcher = new DASearcher(); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java index c8f0d8f9f..aeb402a22 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CommunityDependency.java @@ -24,10 +24,6 @@ import lombok.experimental.Delegate; import org.jboss.pnc.bacon.pig.impl.utils.GAV; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; - /** * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
* Date: 6/1/17 @@ -35,13 +31,12 @@ @Getter @Setter @ToString -public class CommunityDependency { +public class CommunityDependency implements CsvExportable { @Delegate - private final GAV gav; - private String recommendation; - private String availableVersions; - private DependencyState state; - private List usedForSwarm; + final GAV gav; + String recommendation; + String availableVersions; + DependencyState state; /* org/ow2/asm/asm-all/5.0.4/asm-all-5.0.4.jar */ public CommunityDependency(String logLine) { @@ -61,27 +56,18 @@ public String toPathSubstring() { return String.format("%s/%s/", getGroupId().replace('.', '/'), getArtifactId()); } - public void appendToCsv(FileWriter writer) { - try { - writer.append(toCsvLine()).append("\n"); - } catch (IOException e) { - throw new IllegalStateException("Cannot write to the output file", e); - } - } - /** * @return g:a:v; state; recommendation; availableVersions; usedForSwarm */ - private String toCsvLine() { + public String toCsvLine() { return String.format( - "%s:%s:%s; %s; %s; %s; %s", + "%s:%s:%s; %s; %s; %s", getGroupId(), getArtifactId(), getVersion(), state, recommendation, - availableVersions, - usedForSwarm); + availableVersions); } public org.jboss.da.model.rest.GAV toDaGav() { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java new file mode 100644 index 000000000..9c403b9fe --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/CsvExportable.java @@ -0,0 +1,37 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright 2017 Red Hat, Inc., and individual contributors + * as indicated by the @author tags. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da; + +import java.io.FileWriter; +import java.io.IOException; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +public interface CsvExportable { + String toCsvLine(); + + default void appendToCsv(FileWriter writer) { + try { + writer.append(toCsvLine()).append("\n"); + } catch (IOException e) { + throw new IllegalStateException("Cannot write to the output file", e); + } + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java new file mode 100644 index 000000000..b8f728fce --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/documents/sharedcontent/da/ThorntailCommunityDependency.java @@ -0,0 +1,35 @@ +package org.jboss.pnc.bacon.pig.impl.documents.sharedcontent.da; + +import groovy.lang.Delegate; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
+ * Date: 13/08/2019 + */ +@Getter +@Setter +@ToString +public class ThorntailCommunityDependency implements CsvExportable { + + @Delegate + private final CommunityDependency communityDependency; + private List usedForThorntail; + + public ThorntailCommunityDependency(CommunityDependency communityDependency, List usedForThorntail) { + this.communityDependency = communityDependency; + this.usedForThorntail = usedForThorntail; + } + + public String toCsvLine() { + return String.format("%s; %s", communityDependency.toCsvLine(), usedForThorntail); + } + + public List getUsedForThorntail() { + return usedForThorntail; + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java index e6bdb7597..f12c5026b 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java @@ -22,17 +22,21 @@ import org.commonjava.maven.ext.cli.Cli; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.JavadocGenerationData; import org.jboss.pnc.bacon.pig.impl.config.JavadocGenerationStrategy; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Dependency; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Profile; +import org.jboss.pnc.bacon.pig.impl.utils.pom.Project; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +46,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -74,10 +79,16 @@ public class JavadocManager extends DeliverableManager, Void> private File topLevelDirectory; private File archiveFile; private String scmRevision; + private boolean tempBuild; - public JavadocManager(Config config, String releasePath, Deliverables deliverables, Map builds) { - super(config, releasePath, deliverables, builds); - generationData = config.getFlow().getJavadocGeneration(); + public JavadocManager( + PigConfiguration pigConfiguration, + String releasePath, + Deliverables deliverables, + Map builds) { + super(pigConfiguration, releasePath, deliverables, builds); + this.tempBuild = PigProperties.get().isTemporary(); + this.generationData = pigConfiguration.getFlow().getJavadocGeneration(); } public void prepare() { @@ -93,6 +104,7 @@ public void prepare() { break; case IGNORE: log.info("Ignoring javadoc zip generation"); + deliverables.setJavadocZipName(null); break; default: throw new IllegalStateException("Unsupported javadoc generation strategy: " + strategy); @@ -106,7 +118,7 @@ protected void repackage(File contentsDirectory, File targetTopLevelDirectory) { @Override protected String getTargetTopLevelDirectoryName() { - return config.getTopLevelDirectoryPrefix() + "javadoc"; + return pigConfiguration.getTopLevelDirectoryPrefix() + "javadoc"; } @Override @@ -128,20 +140,26 @@ private boolean checkRequired() { } private void init() { - temporaryDestination = FileUtils.mkTempDir("javadoc"); - settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml").getAbsolutePath(); + this.temporaryDestination = FileUtils.mkTempDir("javadoc"); + if (this.tempBuild) { + this.settingsXml = ResourceUtils.extractToTmpFile("/indy-temp-settings.xml", "settings", ".xml") + .getAbsolutePath(); - localRepo = new File(temporaryDestination + File.separator + "localRepo"); - localRepo.mkdir(); - topLevelDirectory = new File(temporaryDestination, getTargetTopLevelDirectoryName()); - archiveFile = getTargetZipPath().toFile(); + } else { + this.settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml") + .getAbsolutePath(); + } + this.localRepo = new File(temporaryDestination + File.separator + "localRepo"); + this.localRepo.mkdir(); + this.topLevelDirectory = new File(temporaryDestination, getTargetTopLevelDirectoryName()); + this.archiveFile = getTargetZipPath().toFile(); - generationProject = generationData.getGenerationProject(); - sourceBuilds = generationData.getSourceBuilds(); - if (sourceBuilds == null || sourceBuilds.isEmpty()) { - sourceBuilds = builds.values().stream().map(PncBuild::getName).collect(Collectors.toList()); + this.generationProject = generationData.getGenerationProject(); + this.sourceBuilds = generationData.getSourceBuilds(); + if (this.sourceBuilds == null || this.sourceBuilds.isEmpty()) { + this.sourceBuilds = builds.values().stream().map(PncBuild::getName).collect(Collectors.toList()); } - scmRevision = generationData.getScmRevision(); + this.scmRevision = generationData.getScmRevision(); } private Collection findSourceBuilds() { @@ -264,9 +282,11 @@ private boolean runPME() { } } System.setOut(outStream); + log.info("PME Command run [{}]", cmd.toString()); if (new Cli().run(cmd.toString().split("\\s+")) != 0) { System.setOut(stdout); log.error("Error running PME see {}", filePath); + dumpLog(filePath); return false; } System.setOut(stdout); @@ -295,17 +315,24 @@ private boolean manipulateProject(Collection srcBuilds) { if (!writeProject(project)) { return false; } else { - runPME(); + return runPME(); } } - return true; + } + + private void dumpLog(String filePath) { + try { + System.out.println(new String(Files.readAllBytes(Paths.get(filePath)))); + } catch (IOException e) { + log.error("Unable to dump log {}", filePath, e); + } } private boolean executeMavenBuild() { log.debug("Executing Javadoc generation maven project"); String command = generationData.getBuildScript(); Process process = null; - if (command == null && command.isEmpty()) { + if (command == null || command.isEmpty()) { // Use a default mvn command on project command = "mvn package -B"; } @@ -320,13 +347,16 @@ private boolean executeMavenBuild() { process.waitFor(); if (process.exitValue() != 0) { log.error("Error while running Javadoc generation project [{}]", process.exitValue()); + dumpLog(mavenRun.getAbsolutePath()); return false; } } catch (IOException e) { log.error("Unable to start build Javadoc generation project", e); + dumpLog(mavenRun.getAbsolutePath()); return false; } catch (InterruptedException e) { log.error("Javadoc generation build was Interrupted", e); + dumpLog(mavenRun.getAbsolutePath()); return false; } return true; diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java index c6c0eb03f..3e808d637 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java @@ -22,15 +22,22 @@ import me.snowdrop.licenses.LicensesGeneratorException; import me.snowdrop.licenses.properties.GeneratorProperties; import me.snowdrop.licenses.utils.Gav; +import org.jboss.pnc.bacon.config.Config; +import org.jboss.pnc.bacon.config.PigConfig; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.util.Collection; import java.util.List; +import java.util.Properties; import java.util.stream.Collectors; /** @@ -47,30 +54,66 @@ public static void generateLicenses(Collection gavs, File archiveFile, Stri File temporaryDestination = FileUtils.mkTempDir("licenses"); File topLevelDirectory = new File(temporaryDestination, topLevelDirectoryName); - generateLicenses(gavs, topLevelDirectory); + generateLicenses(gavs, topLevelDirectory, PigProperties.get().isTemporary()); FileUtils.zip(archiveFile, temporaryDestination, topLevelDirectory); log.debug("Generated zip archive {}", archiveFile); } - private static void generateLicenses(Collection gavs, File temporaryDestination) { + public static void generateLicenses(Collection gavs, File licensesDirectory, boolean useTempBuilds) { try { - LicensesGenerator generator = new LicensesGenerator(prepareGeneratorProperties()); + LicensesGenerator generator = new LicensesGenerator(prepareGeneratorProperties(useTempBuilds)); - generator.generateLicensesForGavs(gavsToLicenseGeneratorGavs(gavs), temporaryDestination.getAbsolutePath()); + generator.generateLicensesForGavs(gavsToLicenseGeneratorGavs(gavs), licensesDirectory.getAbsolutePath()); } catch (LicensesGeneratorException e) { throw new RuntimeException("Failed to generate licenses", e); } } + public static void extractLicenses(File repoZip, File archiveFile, String topLevelDirectoryName) { + File temporaryDestination = FileUtils.mkTempDir("licenses"); + File topLevelDirectory = new File(temporaryDestination, topLevelDirectoryName); + + try { + FileUtils.unzip(repoZip, topLevelDirectory, "^[^/]*/licenses/.*"); + File repoDir = Files.list(topLevelDirectory.toPath()).iterator().next().toFile(); + FileUtils.moveDirectoryContents(new File(repoDir, "licenses"), topLevelDirectory); + org.apache.commons.io.FileUtils.deleteDirectory(repoDir); + FileUtils.zip(archiveFile, temporaryDestination, topLevelDirectory); + log.info("Generated zip archive {}", archiveFile); + } catch (IOException ex) { + throw new RuntimeException("Failed to extract licenses from zip", ex); + } + + } + private static List gavsToLicenseGeneratorGavs(Collection gavs) { return gavs.stream() .map(gav -> new Gav(gav.getGroupId(), gav.getArtifactId(), gav.getVersion(), gav.getPackaging())) .collect(Collectors.toList()); } - private static GeneratorProperties prepareGeneratorProperties() { - File propertiesFile = ResourceUtils - .extractToTmpFile("/license-generator.properties", "license-generator", ".properties"); + private static GeneratorProperties prepareGeneratorProperties(boolean useTempBuilds) { + Properties props = new Properties(); + PigConfig pig = Config.instance().getActiveProfile().getPig(); + String licenseServiceUrl = pig.getLicenseServiceUrl(); + String licenseServiceProp = ""; + if (licenseServiceUrl != null) { + licenseServiceProp = String.format("licenseServiceUrl=%s", licenseServiceUrl); + } + props.setProperty("licenseServiceUrl", licenseServiceProp); + if (useTempBuilds) { + props.setProperty("names", "Indy Temp Builds,Indy Static"); + props.setProperty("urls", Indy.getIndyTempUrl() + "," + Indy.getIndyUrl()); + } else { + props.setProperty("names", "Indy Static"); + props.setProperty("urls", Indy.getIndyUrl()); + } + + File propertiesFile = ResourceUtils.extractToTmpFileWithFiltering( + "/license-generator.properties", + "license-generator", + ".properties", + props); return new GeneratorProperties(propertiesFile.getAbsolutePath()); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java index 919a0241a..ddd7f3798 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java @@ -18,10 +18,12 @@ package org.jboss.pnc.bacon.pig.impl.license; import lombok.Getter; +import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; -import org.jboss.pnc.bacon.pig.impl.config.Config; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.LicenseGenerationStrategy; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; +import org.jboss.pnc.bacon.pig.impl.config.RepoGenerationData; import org.jboss.pnc.bacon.pig.impl.documents.Deliverables; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.repo.RepositoryData; @@ -45,19 +47,25 @@ public class LicenseManager extends DeliverableManager, Void> @Getter private final GenerationData generationData; + private final boolean useTempBuilds; + public LicenseManager( - Config config, + PigConfiguration pigConfiguration, String releasePath, Deliverables deliverables, Map builds, RepositoryData repositoryData) { - super(config, releasePath, deliverables, builds); + super(pigConfiguration, releasePath, deliverables, builds); this.repositoryData = repositoryData; - generationData = config.getFlow().getLicensesGeneration(); + + generationData = pigConfiguration.getFlow().getLicensesGeneration(); + useTempBuilds = PigProperties.get().isTemporary(); } public void prepare() { - switch (generationData.getStrategy()) { + LicenseGenerationStrategy strategy = (generationData == null) ? LicenseGenerationStrategy.IGNORE + : generationData.getStrategy(); + switch (strategy) { case DOWNLOAD: downloadAndRepackage(); break; @@ -79,19 +87,27 @@ protected void repackage(File contentsDirectory, File targetTopLevelDirectory) { @Override protected String getTargetTopLevelDirectoryName() { - return config.getTopLevelDirectoryPrefix() + "licenses"; + return pigConfiguration.getTopLevelDirectoryPrefix() + "licenses"; } @Override protected Path getTargetZipPath() { - return Paths.get(releasePath + deliverables.getLicenseZipName()); + return Paths.get(releasePath, deliverables.getLicenseZipName()); } private void generate() { log.info("Generating licenses"); - LicenseGenerator.generateLicenses( - repositoryData.getGavs(), - getTargetZipPath().toFile(), - getTargetTopLevelDirectoryName()); + RepoGenerationData repoGen = pigConfiguration.getFlow().getRepositoryGeneration(); + if (repoGen.isIncludeLicenses()) { + LicenseGenerator.extractLicenses( + repositoryData.getRepositoryPath().toFile(), + getTargetZipPath().toFile(), + getTargetTopLevelDirectoryName()); + } else { + LicenseGenerator.generateLicenses( + repositoryData.getGavs(), + getTargetZipPath().toFile(), + getTargetTopLevelDirectoryName()); + } } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java index 76f54a9cd..13f14f8d9 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildInfoCollector.java @@ -48,10 +48,10 @@ public class BuildInfoCollector { private final BuildClient buildClient; private final BuildConfigurationClient buildConfigClient; - public void addDependencies(PncBuild bd) { + public void addDependencies(PncBuild bd, String filter) { List artifacts = null; try { - artifacts = toList(buildClient.getDependencyArtifacts(bd.getId())); + artifacts = toList(buildClient.getDependencyArtifacts(bd.getId(), Optional.empty(), Optional.of(filter))); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to get dependency artifacts for " + bd.getId(), e); } @@ -92,8 +92,8 @@ private String readLog(InputStream inputStream) throws IOException { StringBuilder logBuilder = new StringBuilder(); try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream); BufferedReader reader = new BufferedReader(inputStreamReader);) { - reader.lines().forEach(l -> logBuilder.append(l)); - return reader.toString(); + reader.lines().forEach(logBuilder::append); + return logBuilder.toString(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java index b1f227e9e..4dd25c8b0 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java @@ -19,7 +19,7 @@ import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.config.BuildConfig; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.config.ProductConfig; import org.jboss.pnc.bacon.pig.impl.utils.CollectionUtils; import org.jboss.pnc.bacon.pig.impl.utils.PncClientUtils; @@ -84,7 +84,7 @@ public class PncEntitiesImporter { private ProductMilestone milestone; private GroupConfiguration buildGroup; private List configs; - private Config config = PigContext.get().getConfig(); + private PigConfiguration pigConfiguration = PigContext.get().getPigConfiguration(); private PncConfigurator pncConfigurator = new PncConfigurator(); @@ -100,8 +100,10 @@ public PncEntitiesImporter() { public ImportResult performImport() { product = getOrGenerateProduct(); version = getOrGenerateVersion(); - milestone = pncConfigurator - .getOrGenerateMilestone(version, pncMilestoneString(), config.getProduct().getIssueTrackerUrl()); + milestone = pncConfigurator.getOrGenerateMilestone( + version, + pncMilestoneString(), + pigConfiguration.getProduct().getIssueTrackerUrl()); pncConfigurator.markMilestoneCurrent(version, milestone); buildGroup = getOrGenerateBuildGroup(); @@ -222,8 +224,8 @@ private void addConfigurationToGroup(String newConfigId) { private List getAddOrUpdateBuildConfigs() { log.info("Adding/updating build configurations"); List currentConfigs = getCurrentBuildConfigs(); - dropConfigsFromInvalidVersion(currentConfigs, config.getBuilds()); - return updateOrCreate(currentConfigs, config.getBuilds()); + dropConfigsFromInvalidVersion(currentConfigs, pigConfiguration.getBuilds()); + return updateOrCreate(currentConfigs, pigConfiguration.getBuilds()); } private Optional getBuildConfigFromName(String name) { @@ -439,8 +441,9 @@ private GroupConfiguration getOrGenerateBuildGroup() { private Optional getBuildGroup() { try { - return toStream(groupConfigClient.getAll(empty(), Optional.of("name=='" + config.getGroup() + "'"))) - .findAny(); + return toStream( + groupConfigClient.getAll(empty(), Optional.of("name=='" + pigConfiguration.getGroup() + "'"))) + .findAny(); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to check if build group exists"); } @@ -455,7 +458,7 @@ private Product getOrGenerateProduct() { } private Optional getProduct() { - String productName = config.getProduct().getName(); + String productName = pigConfiguration.getProduct().getName(); try { return maybeSingle(productClient.getAll(empty(), findByNameQuery(productName))); } catch (RemoteResourceException e) { @@ -466,7 +469,7 @@ private Optional getProduct() { private ProductVersion generateVersion() { ProductVersion productVersion = ProductVersion.builder() .product(product) - .version(config.getMajorMinor()) + .version(pigConfiguration.getMajorMinor()) .build(); try { return versionClient.createNew(productVersion); @@ -476,7 +479,7 @@ private ProductVersion generateVersion() { } private Product generateProduct() { - ProductConfig productConfig = config.getProduct(); + ProductConfig productConfig = pigConfiguration.getProduct(); Product product = Product.builder() .name(productConfig.getName()) .abbreviation(productConfig.getAbbreviation()) @@ -489,25 +492,28 @@ private Product generateProduct() { } private GroupConfiguration generateBuildGroup(ProductVersionRef version) { - GroupConfiguration group = GroupConfiguration.builder().productVersion(version).name(config.getGroup()).build(); + GroupConfiguration group = GroupConfiguration.builder() + .productVersion(version) + .name(pigConfiguration.getGroup()) + .build(); try { return groupConfigClient.createNew(group); } catch (ClientException e) { - throw new RuntimeException("Failed to create group config: " + config.getGroup()); + throw new RuntimeException("Failed to create group config: " + pigConfiguration.getGroup()); } } public ImportResult readCurrentPncEntities() { - product = getProduct() - .orElseThrow(() -> new RuntimeException("Unable to product " + config.getProduct().getName())); + product = getProduct().orElseThrow( + () -> new RuntimeException("Unable to product " + pigConfiguration.getProduct().getName())); version = getVersion().orElseThrow( () -> new RuntimeException( - "Unable to find version " + config.getMajorMinor() + " for product " + product)); + "Unable to find version " + pigConfiguration.getMajorMinor() + " for product " + product)); milestone = pncConfigurator.getExistingMilestone(version, pncMilestoneString()) .orElseThrow(() -> new RuntimeException("Unable to find milestone " + pncMilestoneString())); // TODO buildGroup = getBuildGroup() - .orElseThrow(() -> new RuntimeException("Unable to find build group " + config.getGroup())); + .orElseThrow(() -> new RuntimeException("Unable to find build group " + pigConfiguration.getGroup())); configs = getBuildConfigs(); @@ -526,7 +532,7 @@ private List getBuildConfigs() { private Optional getVersion() { try { - Optional byName = query("version=='%s'", config.getMajorMinor()); + Optional byName = query("version=='%s'", pigConfiguration.getMajorMinor()); return maybeSingle(productClient.getProductVersions(product.getId(), empty(), byName)); } catch (RemoteResourceException e) { throw new RuntimeException("Failed to query for version", e); @@ -534,6 +540,7 @@ private Optional getVersion() { } private String pncMilestoneString() { - return config.getMajorMinor() + "." + config.getMicro() + "." + config.getMilestone(); + return pigConfiguration.getMajorMinor() + "." + pigConfiguration.getMicro() + "." + + pigConfiguration.getMilestone(); } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java new file mode 100644 index 000000000..7d6dc1643 --- /dev/null +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ArtifactVersion.java @@ -0,0 +1,61 @@ +package org.jboss.pnc.bacon.pig.impl.repo; + +import org.jboss.pnc.bacon.pig.impl.pnc.ArtifactWrapper; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; + +import java.util.Map; +import java.util.Optional; + +/** + * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 + */ +public class ArtifactVersion { + public static String prefix = "artifactVersion://"; + + private ArtifactVersion() { + } + + /* + * gets artifact coordinates from a location of a form artifactVersion://groupId:artifactId[@buildName] and gets a + * version of the artifact from the build + */ + public static String get(String location, String defaultBuildName, Map builds) { + if (!location.startsWith(ArtifactVersion.prefix)) { + throw new RuntimeException("location: " + location + " is not a proper artifactVersion location"); + } + String value = location.substring(prefix.length()); + + String[] split = value.split("@"); + + String gaString = split[0]; + String[] gaSplit = gaString.split(":"); + if (gaSplit.length != 2) { + throw new RuntimeException( + "Expected artifactVersion://groupId:artifactId... as the version locator, got: " + value); + } + + String groupId = gaSplit[0]; + String artifactId = gaSplit[1]; + + String buildName = split.length > 1 ? split[1] : defaultBuildName; + + PncBuild pncBuild = builds.get(buildName); + if (pncBuild == null) { + throw new RuntimeException("Build " + buildName + " not found among the builds"); + } + + Optional maybeArtifact = pncBuild.getBuiltArtifacts().stream().filter(a -> { + GAV gav = a.toGAV(); + return gav.getArtifactId().equals(artifactId) && gav.getGroupId().equals(groupId); + }).findAny(); + + return maybeArtifact + .orElseThrow( + () -> new RuntimeException( + "Unable to find artifact matching " + groupId + ":" + artifactId + + " in artifacts produced by " + buildName)) + .toGAV() + .getVersion(); + } +} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java index a2a4a20ba..8e2b901af 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ExternalArtifactDownloader.java @@ -19,6 +19,9 @@ import org.jboss.pnc.bacon.pig.impl.utils.FileDownloadUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.net.URI; @@ -30,36 +33,38 @@ */ public class ExternalArtifactDownloader { + private static final Logger log = LoggerFactory.getLogger(ExternalArtifactDownloader.class); + private ExternalArtifactDownloader() { } - private static final String TEMPORARY_ARTIFACTS_URL; - private static final String ARTIFACTS_URL; + public static File downloadExternalArtifact(GAV gav, Path targetRepoContents, boolean sourcesOptional) { + File targetPath = targetPath(gav, targetRepoContents); - static { - String indyRepoUrl = System.getenv("INDY_REPO_URL"); - if (indyRepoUrl == null) { - indyRepoUrl = "http://indy.psi.redhat.com/"; - } - if (!indyRepoUrl.endsWith("/")) { - indyRepoUrl = indyRepoUrl + "/"; - } - TEMPORARY_ARTIFACTS_URL = indyRepoUrl + "api/content/maven/group/temporary-builds"; - ARTIFACTS_URL = indyRepoUrl + "api/group/builds-untested+shared-imports+public"; + return downloadExternalArtifact(gav, targetPath, sourcesOptional); } - public static File downloadExternalArtifact(GAV gav, Path targetRepoContents) { - Path versionPath = targetRepoContents.resolve(gav.toVersionPath()); - versionPath.toFile().mkdirs(); - - File targetPath = versionPath.resolve(gav.toFileName()).toFile(); + public static File downloadExternalArtifact(GAV gav, File targetPath, boolean sourcesOptional) { + targetPath.toPath().getParent().toFile().mkdirs(); - String indyUrl = gav.isTemporary() ? TEMPORARY_ARTIFACTS_URL : ARTIFACTS_URL; + String indyUrl = gav.isTemporary() ? Indy.getIndyTempUrl() : Indy.getIndyUrl(); URI downloadUrl = URI.create(String.format("%s/%s", indyUrl, gav.toUri())); - - FileDownloadUtils.downloadTo(downloadUrl, targetPath); + try { + FileDownloadUtils.downloadTo(downloadUrl, targetPath); + } catch (RuntimeException any) { + if (sourcesOptional && "sources".equals(gav.getClassifier()) || "javadoc".equals(gav.getClassifier())) { + log.warn("Unable to download sources for " + gav, any); + } else { + throw any; + } + } return targetPath; } + + public static File targetPath(GAV gav, Path targetRepoContents) { + Path versionPath = targetRepoContents.resolve(gav.toVersionPath()); + return versionPath.resolve(gav.toFileName()).toFile(); + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java index 4afa4cb3e..b81800276 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/ParentPomDownloader.java @@ -1,7 +1,6 @@ package org.jboss.pnc.bacon.pig.impl.repo; -import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; -import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; +import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -21,8 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; +import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; /** * @author Ken Finnigan @@ -31,7 +30,8 @@ public class ParentPomDownloader { public static final Logger log = LoggerFactory.getLogger(ParentPomDownloader.class); - private ParentPomDownloader() { + private ParentPomDownloader(Path repoPath) { + this.repoPath = repoPath; } public static void addParentPoms(Path repoPath) { @@ -40,61 +40,38 @@ public static void addParentPoms(Path repoPath) { } try { - new ParentPomDownloader().process(repoPath); + new ParentPomDownloader(repoPath).process(); } catch (IOException e) { throw new RuntimeException("Unable to download parent poms", e); } } - private void process(final Path repoPath) throws IOException { - Set pomArtifacts = retrievePoms(repoPath); - - File execDir = FileUtils.mkTempDir("parent-pom-retrieval"); - - pomArtifacts.stream().filter(Pom::hasParent).forEach(p -> processPoms(repoPath, p)); - - String settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml").getAbsolutePath(); - toDownload.parallelStream().forEach(gav -> { - log.debug("Downloading: {}", gav.toString()); - // Call Maven to download dependency - - ProcessBuilder builder = new ProcessBuilder( - "mvn", - "-B", - "org.apache.maven.plugins:maven-dependency-plugin:3.0.1:get", - "-Dartifact=" + gav.toString(), - "-Dmaven.repo.local=" + repoPath.toAbsolutePath().toString(), - "-s", - settingsXml); - - builder.directory(execDir).inheritIO(); + private void process() throws IOException { + // iterate until the working set is empty + // this is to download parent POM of a parent POM of a parent POM of a parent POM of a ... + do { + toDownload.clear(); + doProcess(); + } while (!toDownload.isEmpty()); + } - Process process = null; - try { - process = builder.start(); - } catch (IOException e) { - log.error("Unable to download gav {}", gav.toString(), e); - System.out.println(14); - } + private void doProcess() throws IOException { + Files.walk(repoPath) + .filter(Files::isRegularFile) + .filter(ParentPomDownloader::isPom) + .map(Pom::new) + .filter(Pom::hasParent) + .forEach(p -> processPom(repoPath, p)); - while (process.isAlive()) { - try { - Thread.sleep(1000); - if (process.exitValue() == 0) { - break; - } - } catch (IllegalThreadStateException e) { - // ignore as process not exited - } catch (InterruptedException e) { - break; - } - } - } + toDownload.parallelStream().map(PomGAV::toGav).forEach(this::download); + } - ); + private void download(GAV gav) { + log.info("Downloading missing parent POM {}", gav); + ExternalArtifactDownloader.downloadExternalArtifact(gav, repoPath, false); } - private void processPoms(Path repoPath, Pom pom) { + private void processPom(Path repoPath, Pom pom) { PomGAV coords = parentCoordinates(pom); if (!coords.version.contains("redhat")) { // community parent POM not required @@ -102,11 +79,12 @@ private void processPoms(Path repoPath, Pom pom) { } Path parentDir = artifactDir(repoPath, coords); - Path parentPomPath = parentDir.resolve(coords.artifactId() + "-" + coords.version() + ".pom"); + Path parentPomPath = parentDir.resolve(coords.artifactId + "-" + coords.version + ".pom"); if (!alreadyChecked.contains(coords)) { if (!Files.isRegularFile(parentPomPath)) { // File missing + log.debug("Will download {} because it's a parent of {}", coords, pom.path); toDownload.add(coords); } @@ -114,22 +92,14 @@ private void processPoms(Path repoPath, Pom pom) { } } - private Set retrievePoms(Path repoPath) throws IOException { - return Files.walk(repoPath) - .filter(Files::isRegularFile) - .filter(ParentPomDownloader::isPom) - .map(Pom::new) - .collect(Collectors.toSet()); - } - private static boolean isPom(Path path) { return path.toString().endsWith(".pom"); } private static Path artifactDir(Path repoPath, PomGAV coords) { - Path groupDir = repoPath.resolve(coords.groupId().replace('.', File.separatorChar)); - Path artifactDir = groupDir.resolve(coords.artifactId()); - return artifactDir.resolve(coords.version()); + Path groupDir = repoPath.resolve(coords.groupId.replace('.', File.separatorChar)); + Path artifactDir = groupDir.resolve(coords.artifactId); + return artifactDir.resolve(coords.version); } private PomGAV parentCoordinates(Pom pom) { @@ -174,9 +144,11 @@ private XPathExpression parentVersionExpression() throws XPathExpressionExceptio return parentVersionExpression; } - private Set alreadyChecked = new HashSet<>(); + private final Set alreadyChecked = new HashSet<>(); + + private final Set toDownload = new HashSet<>(); - private Set toDownload = new HashSet<>(); + private final Path repoPath; private XPath xpath; @@ -186,7 +158,7 @@ private XPathExpression parentVersionExpression() throws XPathExpressionExceptio private XPathExpression parentVersionExpression; - private class Pom { + private static class Pom { private final Path path; private DocumentBuilder documentBuilder; @@ -220,7 +192,7 @@ private DocumentBuilder documentBuilder() throws ParserConfigurationException { } } - private class PomGAV { + private static class PomGAV { private final String groupId; private final String artifactId; @@ -233,21 +205,29 @@ private class PomGAV { this.version = version; } - String groupId() { - return groupId; + @Override + public String toString() { + return groupId + ':' + artifactId + ':' + version + ":pom"; } - String artifactId() { - return artifactId; + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof PomGAV)) + return false; + PomGAV pomGAV = (PomGAV) o; + return Objects.equals(groupId, pomGAV.groupId) && Objects.equals(artifactId, pomGAV.artifactId) + && Objects.equals(version, pomGAV.version); } - String version() { - return version; + @Override + public int hashCode() { + return Objects.hash(groupId, artifactId, version); } - @Override - public String toString() { - return groupId + ':' + artifactId + ':' + version + ":pom"; + public GAV toGav() { + return new GAV(groupId, artifactId, version, "pom"); } } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java index 3f5428712..d5b3d8aa0 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java @@ -20,12 +20,15 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.jboss.pnc.bacon.pig.impl.config.Config; +import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.config.RepoGenerationData; +import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.OSCommandExecutor; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; import org.jboss.pnc.bacon.pig.impl.utils.XmlUtils; +import org.jboss.pnc.bacon.pig.impl.utils.indy.Indy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; @@ -36,8 +39,12 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.apache.commons.lang3.StringUtils.isEmpty; + /** * TODO: replace with programmatic maven api to remove OSCommandExecutor? TODO: CON: would mean that users have to stick * to the same maven version @@ -50,46 +57,95 @@ public class RepoBuilder { private static final String POM_TEMPLATE_LOCATION = "/pom-template.xml"; private static final String ENCODING = "UTF-8"; - private final Config config; + private final PigConfiguration pigConfiguration; private final boolean removeGeneratedM2Dups; + private final boolean tempBuild; + private final RepoGenerationData repoGeneration; + private final Map builds; + private final String additionalRepo; String topLevelDirectoryName; Path configurationDirectory; - public RepoBuilder(Config config, Path configurationDirectory, boolean removeGeneratedM2Dups) { - this.config = config; - topLevelDirectoryName = config.getTopLevelDirectoryPrefix() + "maven-repository"; + public RepoBuilder( + PigConfiguration pigConfiguration, + String additionalRepo, + Path configurationDirectory, + Map builds) { + this.pigConfiguration = pigConfiguration; + this.additionalRepo = additionalRepo; + this.builds = builds; this.configurationDirectory = configurationDirectory; - this.removeGeneratedM2Dups = removeGeneratedM2Dups; + + repoGeneration = pigConfiguration.getFlow().getRepositoryGeneration(); + removeGeneratedM2Dups = PigProperties.get().isRemoveGeneratedM2Dups(); + tempBuild = PigProperties.get().isTemporary(); + topLevelDirectoryName = pigConfiguration.getTopLevelDirectoryPrefix() + "maven-repository"; } - public File build(File bomFile) { + public void build(File bomFile, File repoParentDir, Predicate artifactSelector) { try { - File projectLocation = createProject(bomFile); - String settingsXml = ResourceUtils.extractToTmpFile("/indy-settings.xml", "settings", ".xml") - .getAbsolutePath(); - File m2Repo = buildProjectWithOverriddenM2(projectLocation, settingsXml); - RepositoryUtils.removeIrrelevantFiles(m2Repo); + createAndBuildProject(bomFile, repoParentDir, artifactSelector); + RepositoryUtils.removeIrrelevantFiles(repoParentDir); if (removeGeneratedM2Dups) { - RepositoryUtils.keepOnlyLatestRedHatArtifacts(m2Repo); + RepositoryUtils.keepOnlyLatestRedHatArtifacts(repoParentDir); } - return m2Repo; } catch (IOException e) { throw new RuntimeException("Unable to build pom", e); } } - private File buildProjectWithOverriddenM2(File projectLocation, String settingsXml) { + private void createAndBuildProject(File bomFile, File repoParentDir, Predicate artifactSelector) + throws IOException { + File projectLocation = createProject(bomFile, artifactSelector); + String indySettings; + if (tempBuild) { + indySettings = "/indy-temp-settings.xml"; + } else { + indySettings = "/indy-settings.xml"; + } + String settingsXml = ResourceUtils + .extractToTmpFileWithFiltering( + indySettings, + "settings", + ".xml", + settingsProps(tempBuild, additionalRepo)) + .getAbsolutePath(); + + File repoDir = new File(repoParentDir, RepoDescriptor.MAVEN_REPOSITORY); + + buildProjectWithOverriddenM2(projectLocation, repoDir, settingsXml); + } + + private Properties settingsProps(boolean tempBuild, String additionalRepo) { + Properties result = new Properties(); + if (tempBuild) { + String repoDef = "--> \n" + " additional\n" + " " + + additionalRepo + "\n" + " \n" + " additional-plugins\n" + + " " + additionalRepo + "\n" + " + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + indy-repositories + + + indy + ${INDY_URL} + + + + + indy + ${INDY_URL} + + + + + + indy-repositories + maven-central + + diff --git a/pig/src/main/resources/indy-temp-settings.xml b/pig/src/main/resources/indy-temp-settings.xml new file mode 100644 index 000000000..8cb2594a4 --- /dev/null +++ b/pig/src/main/resources/indy-temp-settings.xml @@ -0,0 +1,69 @@ + + + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + indy-repositories + + + indy + ${INDY_URL} + + + indy-temp + ${INDY_TMP_URL}/ + + + + + + indy + ${INDY_URL} + + + indy-temp + ${INDY_TMP_URL} + + + + + + + + + indy-repositories + maven-central + + + diff --git a/pig/src/main/resources/koji-build-finder/config.json b/pig/src/main/resources/koji-build-finder/config.json new file mode 100644 index 000000000..631bbfa91 --- /dev/null +++ b/pig/src/main/resources/koji-build-finder/config.json @@ -0,0 +1,10 @@ +{ + "archive-extensions" : [ "dll", "dylib", "ear", "jar", "jdocbook", "jdocbook-style", "kar", "plugin", "pom", "rar", "sar", "so", "war", "xml" ], + "archive-types" : [ "jar", "xml", "pom", "so", "dll", "dylib" ], + "checksum-only" : false, + "checksum-type" : "md5", + "excludes" : [ "^(?!.*/pom\\.xml$).*/.*\\.xml$" ], + "koji-hub-url" : "${KOJI_URL}/brewhub", + "koji-web-url" : "${KOJI_URL}/brew" +} + diff --git a/pig/src/main/resources/license-generator.properties b/pig/src/main/resources/license-generator.properties new file mode 100644 index 000000000..08635de94 --- /dev/null +++ b/pig/src/main/resources/license-generator.properties @@ -0,0 +1,4 @@ +# to run with PNC stage, you need to set the `INDY_REPO_URL` environment variable to http://indy-stage.psi.redhat.com/ +repository.names=${names} +repository.urls=${urls} +$licenseServiceUrl \ No newline at end of file diff --git a/pig/src/main/resources/pom-template.xml b/pig/src/main/resources/pom-template.xml new file mode 100644 index 000000000..a3de5e71b --- /dev/null +++ b/pig/src/main/resources/pom-template.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + com.example + user-project + 1.0-SNAPSHOT + + + + + + + + import + pom + + + + + + + + + + maven-dependency-plugin + org.apache.maven.plugins + 2.10 + + + download-sources + prepare-package + + sources + + + + + + + + + 1.8 + 1.8 + + diff --git a/pig/src/main/resources/release.sh b/pig/src/main/resources/release.sh new file mode 100644 index 000000000..2f4cfd7f5 --- /dev/null +++ b/pig/src/main/resources/release.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e +# mstodo: + +<#noparse> +function getPushStatus() { + pnc brew-push status $1 | grep "status" | cut -d : -f 2 | tr -d '[:space:]' | tr -d '"' +} + +function waitForPush() { + while true + do + status=$(getPushStatus $1) + if [ "${status}" == "SUCCESS" ] + then + break; + fi + + if [ "${status}" == "SYSTEM_ERROR" ] || [ "${status}" == "CANCELED" ] || [ "${status}" == "FAILED" ] + then + echo "Failed to push build $1 to Brew" + exit 1 + fi + echo "Waiting for build $1 to be pushed to Brew" + sleep 5 + done +} + + +<#list buildsToPush as build> +pnc brew-push build ${build} ${brewTag} +waitForPush ${build} + + +/bin/bash ${nvrListScriptLocation} ${repoZipLocation} ${targetPath} ${kojiHubUrl} + +pnc close-milestone ${milestoneId} --wait diff --git a/pig/src/main/resources/repository-README.md b/pig/src/main/resources/repository-README.md new file mode 100644 index 000000000..804179897 --- /dev/null +++ b/pig/src/main/resources/repository-README.md @@ -0,0 +1,4 @@ +${PRODUCT_NAME} Maven Repository +======================= + +This archive contains Maven repository artifacts for ${PRODUCT_NAME}. diff --git a/pig/src/main/resources/repository-example-settings.xml b/pig/src/main/resources/repository-example-settings.xml new file mode 100644 index 000000000..5c5154d93 --- /dev/null +++ b/pig/src/main/resources/repository-example-settings.xml @@ -0,0 +1,54 @@ + + + + + + + repository + + + repository + file:///path/to/repo/repository/maven-repository + + true + + + false + + + + + + repository + file:///path/to/repo/repository/maven-repository + + true + + + false + + + + + + + + repository + + \ No newline at end of file diff --git a/pig/src/main/resources/rh-license-exceptions.json b/pig/src/main/resources/rh-license-exceptions.json new file mode 100644 index 000000000..af1fbc80e --- /dev/null +++ b/pig/src/main/resources/rh-license-exceptions.json @@ -0,0 +1,1113 @@ +[ + { + "groupId": "org.apache.tomcat", + "artifactId": "servlet-api", + "version": "6.0.41", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + { + "name": "Common Development And Distribution License (CDDL) Version 1.0", + "url": "http://www.opensource.org/licenses/cddl1.txt" + } + ] + }, + { + "groupId": "com.squareup.okhttp3", + "artifactId": "benchmarks", + "version": "3.9.0.redhat-3", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.tomcat", + "artifactId": "servlet-api", + "version": "6.0.41-redhat", + "licenses": [ + { + "name": "Apache License, Version 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + { + "name": "Common Development And Distribution License (CDDL) Version 1.0", + "url": "http://www.opensource.org/licenses/cddl1.txt" + } + ] + }, + { + "groupId": "antlr", + "artifactId": "antlr", + "version": "2.7.7", + "licenses": [ + { + "name": "The Antlr 2.7.7 License", + "url": "http://www.antlr2.org/license.html" + } + ] + }, + { + "groupId": "antlr", + "artifactId": "antlr", + "version": "2.7.7-redhat", + "licenses": [ + { + "name": "The Antlr 2.7.7 License", + "url": "http://www.antlr2.org/license.html" + } + ] + }, + { + "groupId": "dom4j", + "artifactId": "dom4j", + "version": "1.6.1", + "licenses": [ + { + "name": "Plexus Classworlds License", + "url": "https://raw.githubusercontent.com/dom4j/dom4j/dom4j_1_6_1/LICENSE.txt" + } + ] + }, + { + "groupId": "dom4j", + "artifactId": "dom4j", + "version": "1.6.1-redhat", + "licenses": [ + { + "name": "Plexus Classworlds License", + "url": "https://raw.githubusercontent.com/dom4j/dom4j/dom4j_1_6_1/LICENSE.txt" + } + ] + }, + { + "groupId": "com.h2database", + "artifactId": "h2", + "version": "1.4.196", + "licenses": [ + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + }, + { + "name": "Mozilla Public License v2.0", + "url": "http://www.mozilla.org/MPL/2.0" + } + ] + }, + { + "groupId": "org.jboss.openjdk-orb", + "artifactId": "openjdk-orb", + "version": "8.0.8.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.rmi", + "artifactId": "jboss-rmi-api_1.0_spec", + "version": "1.0.5.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ejb", + "artifactId": "jboss-ejb-api_3.2_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.2_spec", + "version": "2.2.13.redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.interceptor", + "artifactId": "jboss-interceptors-api_1.2_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.jms", + "artifactId": "jboss-jms-api_2.0_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.resource", + "artifactId": "jboss-connector-api_1.7_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.security.jacc", + "artifactId": "jboss-jacc-api_1.5_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.transaction", + "artifactId": "jboss-transaction-api_1.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ws.rs", + "artifactId": "jboss-jaxrs-api_2.0_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.bind", + "artifactId": "jboss-jaxb-api_2.2_spec", + "version": "1.0.4.Final-redhat-4", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.rpc", + "artifactId": "jboss-jaxrpc-api_1.1_spec", + "version": "1.0.1.Final-redhat-5", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.soap", + "artifactId": "jboss-saaj-api_1.3_spec", + "version": "1.0.4.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.ws", + "artifactId": "jboss-jaxws-api_2.2_spec", + "version": "2.0.4.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.2_spec", + "version": "2.2.13.SP2-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.4.redhat-7", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.4.redhat-7", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.reactivex", + "artifactId": "rxjava", + "version": "1.3.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.jboss.openjdk-orb", + "artifactId": "openjdk-orb", + "version": "8.1.2.Final-redhat-00001", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ejb", + "artifactId": "jboss-ejb-api_3.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.faces", + "artifactId": "jboss-jsf-api_2.3_spec", + "version": "2.3.5.SP1-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.interceptor", + "artifactId": "jboss-interceptors-api_1.2_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.jms", + "artifactId": "jboss-jms-api_2.0_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.resource", + "artifactId": "jboss-connector-api_1.7_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.security.jacc", + "artifactId": "jboss-jacc-api_1.5_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.transaction", + "artifactId": "jboss-transaction-api_1.2_spec", + "version": "1.1.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.ws.rs", + "artifactId": "jboss-jaxrs-api_2.1_spec", + "version": "1.0.1.Final-redhat-00001", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.bind", + "artifactId": "jboss-jaxb-api_2.3_spec", + "version": "1.0.1.Final-redhat-1", + "licenses": [ + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + }, + { + "name": "Common Development and Distribution License 1.1", + "url": "https://javaee.github.io/glassfish/LICENSE" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.rpc", + "artifactId": "jboss-jaxrpc-api_1.1_spec", + "version": "1.0.2.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.soap", + "artifactId": "jboss-saaj-api_1.3_spec", + "version": "1.0.6.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.jboss.spec.javax.xml.ws", + "artifactId": "jboss-jaxws-api_2.3_spec", + "version": "1.0.0.Final-redhat-1", + "licenses": [ + { + "name": "Common Development and Distribution License 1.0", + "url": "http://repository.jboss.org/licenses/cddl.txt" + }, + { + "name": "GNU General Public License v2.0 only, with Classpath exception", + "url": "http://repository.jboss.org/licenses/gpl-2.0-ce.txt" + } + ] + }, + { + "groupId": "org.apache.ant", + "artifactId": "ant-parent", + "version": "1.9.4.redhat-1", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.ant", + "artifactId": "ant-launcher", + "version": "1.9.4.redhat-1", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.10.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-compiler", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.8.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.10.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-library", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.scala-lang", + "artifactId": "scala-reflect", + "version": "2.12.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.1.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.1.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.2.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.2.1.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.3.0.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.3.0.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.3.0.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.3.0.redhat-00003", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "12.0.0.redhat-00002", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "13.0.0.redhat-00001", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.graphql-java", + "artifactId": "graphql-java", + "version": "14.0.0.redhat-00001", + "licenses": [ + { + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT" + } + ] + }, + { + "groupId": "com.fasterxml.jackson.module", + "artifactId": "jackson-module-scala_2.12", + "version": "2.9.9.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "com.fasterxml.jackson.module", + "artifactId": "jackson-module-scala_2.12", + "version": "2.9.10.redhat-00002", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.reactivestreams", + "artifactId": "reactive-streams", + "version-regexp": ".*", + "licenses": [ + { + "name": "Creative Commons Zero 1.0 Universal", + "url": "http://creativecommons.org/publicdomain/zero/1.0/legalcode" + } + ] + }, + { + "groupId": "com.thoughtworks.paranamer", + "artifactId": "paranamer-parent", + "version": "2.8.0.redhat-00001", + "licenses": [ + { + "name": "BSD 3-clause \"New\" or \"Revised\" License", + "url": "https://opensource.org/licenses/BSD-3-Clause" + } + ] + }, + { + "groupId": "com.thoughtworks.paranamer", + "artifactId": "paranamer", + "version": "2.8.0.redhat-00001", + "licenses": [ + { + "name": "BSD 3-clause \"New\" or \"Revised\" License", + "url": "https://opensource.org/licenses/BSD-3-Clause" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "btf", + "version": "1.2.0.redhat-00006", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "jackson-coreutils", + "version": "1.6.0.redhat-00006", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "com.github.fge", + "artifactId": "msg-simple", + "version": "1.1.0.redhat-00005", + "licenses": [ + { + "name": "GNU Lesser General Public License v3.0 or later", + "url": "https://spdx.org/licenses/LGPL-3.0+.html" + }, + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + } + ] + }, + { + "groupId": "io.reactivex.rxjava2", + "artifactId": "rxjava", + "version": "2.2.18.redhat-00001", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka-clients", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.apache.kafka", + "artifactId": "kafka_2.12", + "version": "2.4.0.redhat-00005", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "javax.persistence", + "artifactId": "javax.persistence-api", + "version": "2.2.0.redhat-1", + "licenses": [ + { + "name": "Eclipse Distribution License, Version 1.0", + "url": "https://repository.jboss.org/licenses/edl-1.0.txt" + }, + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-client", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-core", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-thrift", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "io.jaegertracing", + "artifactId": "jaeger-tracerresolver", + "version-regexp": "1.1.*", + "licenses": [ + { + "name": "Apache License 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ] + }, + { + "groupId": "org.eclipse", + "artifactId": "yasson", + "version-regexp": ".*", + "licenses": [ + { + "name": "Eclipse Distribution License, Version 1.0", + "url": "https://repository.jboss.org/licenses/edl-1.0.txt" + }, + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + }, + { + "groupId": "org.eclipse.jdt.core.compiler", + "artifactId": "ecj", + "version-range": "[4.0,5.0)", + "licenses": [ + { + "name": "Eclipse Public License, Version 1.0", + "url": "http://repository.jboss.org/licenses/epl-1.0.txt" + } + ] + } +] diff --git a/pig/src/main/resources/settings-template.xml b/pig/src/main/resources/settings-template.xml new file mode 100644 index 000000000..14cb3c5f6 --- /dev/null +++ b/pig/src/main/resources/settings-template.xml @@ -0,0 +1,57 @@ + + + + + ${user.home}/.m2/repository + + + maven-central + + + central + https://repo1.maven.org/maven2/ + + + + + central + https://repo1.maven.org/maven2/ + + + + + additional-repo + + + additional-repo + ADD_REPO_URL + + + + + additional-repo + ADD_REPO_URL + + + + + + additional-repo + maven-central + + From 5a8273781a1c7664bf61ef7bf3668f590492d1b0 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Mon, 8 Jun 2020 14:08:46 +0100 Subject: [PATCH 3/5] Fix Javadoc yaml config --- .../pnc/bacon/pig/impl/config/JavadocGenerationData.java | 2 +- .../org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java index 16352e95d..772d31c88 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java @@ -34,6 +34,6 @@ public class JavadocGenerationData extends GenerationData alignmentParameters = new TreeSet<>(); + private Set customPmeParameters = new TreeSet<>(); private String importBom; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java index f12c5026b..36e06b4fc 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java @@ -276,8 +276,8 @@ private boolean runPME() { "-f " + topLevelDirectory.getPath() + File.separator + "pom.xml" + " -DprofileInjection=" + project_gid + ":" + project_aid + ":" + project_version + " -s " + settingsXml + " -Dmaven.repo.local=" + localRepo + " -t"); - if (generationData.getAlignmentParameters() != null && !generationData.getAlignmentParameters().isEmpty()) { - for (String parameter : generationData.getAlignmentParameters()) { + if (generationData.getCustomPmeParameters() != null && !generationData.getCustomPmeParameters().isEmpty()) { + for (String parameter : generationData.getCustomPmeParameters()) { cmd.append(" " + parameter); } } From c67b9ba1492c88236421d2a10705bfb79da1a3e8 Mon Sep 17 00:00:00 2001 From: Michal Szynkiewicz Date: Mon, 8 Jun 2020 15:37:30 +0200 Subject: [PATCH 4/5] eliminated `PigProperties` --- .../java/org/jboss/pnc/bacon/pig/Pig.java | 48 ++++++++++++------- .../org/jboss/pnc/bacon/pig/PigFacade.java | 10 ++-- .../jboss/pnc/bacon/pig/impl/PigContext.java | 2 + .../pnc/bacon/pig/impl/PigProperties.java | 37 -------------- .../bacon/pig/impl/config/BuildConfig.java | 9 ++-- .../pig/impl/javadoc/JavadocManager.java | 8 ++-- .../pig/impl/license/LicenseGenerator.java | 4 +- .../pig/impl/license/LicenseManager.java | 4 +- .../bacon/pig/impl/pnc/BuildConfigData.java | 4 +- .../pig/impl/pnc/PncEntitiesImporter.java | 21 ++++---- .../pnc/bacon/pig/impl/repo/RepoBuilder.java | 9 ++-- .../pnc/bacon/pig/impl/repo/RepoManager.java | 7 +-- 12 files changed, 73 insertions(+), 90 deletions(-) delete mode 100644 pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java index 87a8d2225..a122b6639 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java @@ -62,10 +62,23 @@ public class Pig extends AbstractCommand { public static final String REMOVE_M2_DUPLICATES = "removeGeneratedM2Dups"; public static final String REMOVE_M2_DUPLICATES_DEFAULT = "false"; + public static final String SKIP_BRANCH_CHECK = "skipBranchCheck"; + public static final String SKIP_BRANCH_CHECK_DEFAULT = "false"; + public static final String SKIP_BRANCH_CHECK_DESC = "If set to true, pig won't try to determine if the branch that is used to build from is modified. " + + "Branch modification check takes a lot of time, if you use tag, this switch can speed up the build."; + public abstract class PigCommand extends AbstractCommand { @Argument(required = true, description = "Directory containing the Pig configuration file") String configDir; + @Option( + shortName = TEMP_BUILD_SHORT, + name = TEMP_BUILD, + overrideRequired = true, + defaultValue = TEMP_BUILD_DEFAULT, + description = TEMP_BUILD_DESC) + boolean tempBuild; // mstodo add support for specifying tempBuild for more commands + @Option( shortName = 'o', overrideRequired = false, @@ -96,14 +109,6 @@ public class Run extends PigCommand { // description = "id of the group to build. Exactly one of {config, build-group} has to be provided") // private Integer buildGroupId; - @Option( - shortName = TEMP_BUILD_SHORT, - name = TEMP_BUILD, - overrideRequired = true, - defaultValue = TEMP_BUILD_DEFAULT, - description = TEMP_BUILD_DESC) - private boolean tempBuild; - @Option( name = TEMP_BUILD_TIME_STAMP, overrideRequired = true, @@ -175,6 +180,13 @@ public class Run extends PigCommand { description = REMOVE_M2_DUPLICATES_DESC) private boolean removeGeneratedM2Dups; + @Option( + name = SKIP_BRANCH_CHECK, + overrideRequired = true, + defaultValue = SKIP_BRANCH_CHECK_DEFAULT, + description = SKIP_BRANCH_CHECK_DESC) + private boolean skipBranchCheck; + @Option( shortName = 'r', name = "repoZip", @@ -198,16 +210,24 @@ public String doExecute() { repoZipPath, tempBuild, tempBuildTS, - rebuildMode); + rebuildMode, + skipBranchCheck); } } @CommandDefinition(name = "configure", description = "Configure PNC entities") public class Configure extends PigCommand { + @Option( + name = SKIP_BRANCH_CHECK, + overrideRequired = true, + defaultValue = SKIP_BRANCH_CHECK_DEFAULT, + description = SKIP_BRANCH_CHECK_DESC) + private boolean skipBranchCheck; + @Override public ImportResult doExecute() { - ImportResult importResult = PigFacade.importPncEntities(); + ImportResult importResult = PigFacade.importPncEntities(skipBranchCheck); PigContext.get().setPncImportResult(importResult); return importResult; } @@ -222,14 +242,6 @@ public class Build extends PigCommand> { // description = "id of the group to build. Exactly one of {config, build-group} has to be provided") // private Integer buildGroupId; - @Option( - shortName = TEMP_BUILD_SHORT, - name = TEMP_BUILD, - overrideRequired = true, - defaultValue = TEMP_BUILD_DEFAULT, - description = TEMP_BUILD_DESC) - private boolean tempBuild; - @Option( name = TEMP_BUILD_TIME_STAMP, overrideRequired = true, diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java index 68ac628a4..03c1fbe15 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/PigFacade.java @@ -67,9 +67,9 @@ public class PigFacade { private PigFacade() { } - public static ImportResult importPncEntities() { + public static ImportResult importPncEntities(boolean skipBranchCheck) { PncEntitiesImporter pncImporter = new PncEntitiesImporter(); - return pncImporter.performImport(); + return pncImporter.performImport(skipBranchCheck); } public static ImportResult readPncEntities() { @@ -78,6 +78,7 @@ public static ImportResult readPncEntities() { } public static Map build(boolean tempBuild, boolean tempBuildTS, RebuildMode rebuildMode) { + context().setTempBuild(tempBuild); ImportResult importResult = context().getPncImportResult(); if (importResult == null) { importResult = readPncEntities(); @@ -103,7 +104,8 @@ public static String run( String repoZipPath, boolean tempBuild, boolean tempBuildTS, - RebuildMode rebuildMode) { + RebuildMode rebuildMode, + boolean skipBranchCheck) { PigContext context = context(); @@ -111,7 +113,7 @@ public static String run( if (skipPncUpdate) { importResult = readPncEntities(); } else { - importResult = importPncEntities(); + importResult = importPncEntities(skipBranchCheck); } context.setPncImportResult(importResult); context.storeContext(); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java index 2a2d1ddee..52c19f18e 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigContext.java @@ -61,6 +61,8 @@ public class PigContext { private String releasePath; private String extrasPath; + private boolean tempBuild; + public void setPigConfiguration(PigConfiguration pigConfiguration) { this.pigConfiguration = pigConfiguration; if (deliverables == null) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java deleted file mode 100644 index 245a6f968..000000000 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/PigProperties.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.jboss.pnc.bacon.pig.impl; - -import lombok.Getter; - -import java.util.Properties; - -/** - * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com 2020-06-05 - */ -@Getter -public class PigProperties { - - private boolean temporary; - private boolean removeGeneratedM2Dups; - private boolean skipBranchCheck; - - private PigProperties() { - } - - private static PigProperties instance; - - public static synchronized void init(Properties properties) { - instance = new PigProperties(); - instance.removeGeneratedM2Dups = Boolean.TRUE.toString() - .equalsIgnoreCase(properties.getProperty("removeGeneratedM2Dups", "false")); - instance.skipBranchCheck = Boolean.TRUE.toString() - .equalsIgnoreCase(properties.getProperty("skipBranchCheck", "false")); - instance.temporary = Boolean.TRUE.toString().equalsIgnoreCase(properties.getProperty("temporary", "false")); - } - - public static synchronized PigProperties get() { - if (instance == null) { - init(System.getProperties()); - } - return instance; - } -} diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java index 99480d4bc..d29a03905 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/BuildConfig.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; -import org.jboss.pnc.bacon.pig.impl.PigProperties; import org.jboss.pnc.bacon.pig.impl.pnc.GitRepoInspector; import org.jboss.pnc.dto.BuildConfiguration; import org.jboss.pnc.dto.SCMRepository; @@ -124,8 +123,8 @@ public void sanitizebuildScript() { } @JsonIgnore - public synchronized boolean isBranchModified(BuildConfiguration oldVersion) { - if (PigProperties.get().isSkipBranchCheck()) { + public synchronized boolean isBranchModified(BuildConfiguration oldVersion, boolean skipBranchCheck) { + if (skipBranchCheck) { return false; } if (branchModified == null) { @@ -138,7 +137,7 @@ public synchronized boolean isBranchModified(BuildConfiguration oldVersion) { } @JsonIgnore - public boolean isTheSameAs(BuildConfiguration old) { + public boolean isTheSameAs(BuildConfiguration old, boolean skipBranchCheck) { return old != null && StringUtils.equals(name, old.getName()) && StringUtils.equals(project, old.getProject().getName()) && StringUtils.equals(buildScript, old.getBuildScript()) @@ -146,7 +145,7 @@ public boolean isTheSameAs(BuildConfiguration old) { && StringUtils.equals(scmRevision, old.getScmRevision()) && environmentId.equals(old.getEnvironment().getId()) && alignmentParameters.equals(getAlignmentParameters(old)) && urlsEqual(old.getScmRepository()) - && !isBranchModified(old); + && !isBranchModified(old, skipBranchCheck); } private Set getAlignmentParameters(BuildConfiguration old) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java index f12c5026b..adcdf9428 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java @@ -22,7 +22,7 @@ import org.commonjava.maven.ext.cli.Cli; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; -import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.JavadocGenerationData; @@ -70,6 +70,7 @@ public class JavadocManager extends DeliverableManager, Void> @Getter private final JavadocGenerationData generationData; + private final boolean tempBuild; private String generationProject; private List sourceBuilds; @@ -79,7 +80,6 @@ public class JavadocManager extends DeliverableManager, Void> private File topLevelDirectory; private File archiveFile; private String scmRevision; - private boolean tempBuild; public JavadocManager( PigConfiguration pigConfiguration, @@ -87,8 +87,8 @@ public JavadocManager( Deliverables deliverables, Map builds) { super(pigConfiguration, releasePath, deliverables, builds); - this.tempBuild = PigProperties.get().isTemporary(); - this.generationData = pigConfiguration.getFlow().getJavadocGeneration(); + tempBuild = PigContext.get().isTempBuild(); + generationData = pigConfiguration.getFlow().getJavadocGeneration(); } public void prepare() { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java index 3e808d637..48e43d181 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseGenerator.java @@ -24,7 +24,7 @@ import me.snowdrop.licenses.utils.Gav; import org.jboss.pnc.bacon.config.Config; import org.jboss.pnc.bacon.config.PigConfig; -import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.utils.FileUtils; import org.jboss.pnc.bacon.pig.impl.utils.GAV; import org.jboss.pnc.bacon.pig.impl.utils.ResourceUtils; @@ -54,7 +54,7 @@ public static void generateLicenses(Collection gavs, File archiveFile, Stri File temporaryDestination = FileUtils.mkTempDir("licenses"); File topLevelDirectory = new File(temporaryDestination, topLevelDirectoryName); - generateLicenses(gavs, topLevelDirectory, PigProperties.get().isTemporary()); + generateLicenses(gavs, topLevelDirectory, PigContext.get().isTempBuild()); FileUtils.zip(archiveFile, temporaryDestination, topLevelDirectory); log.debug("Generated zip archive {}", archiveFile); } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java index ddd7f3798..fcfca8b15 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/license/LicenseManager.java @@ -18,7 +18,7 @@ package org.jboss.pnc.bacon.pig.impl.license; import lombok.Getter; -import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; import org.jboss.pnc.bacon.pig.impl.config.GenerationData; import org.jboss.pnc.bacon.pig.impl.config.LicenseGenerationStrategy; @@ -59,7 +59,7 @@ public LicenseManager( this.repositoryData = repositoryData; generationData = pigConfiguration.getFlow().getLicensesGeneration(); - useTempBuilds = PigProperties.get().isTemporary(); + useTempBuilds = PigContext.get().isTempBuild(); } public void prepare() { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildConfigData.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildConfigData.java index da0aa7e34..14fe3bc6a 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildConfigData.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/BuildConfigData.java @@ -41,8 +41,8 @@ public BuildConfigData(BuildConfig newConfig) { this.newConfig = newConfig; } - public boolean shouldBeUpdated() { - return !newConfig.isTheSameAs(oldConfig); + public boolean shouldBeUpdated(boolean skipBranchCheck) { + return !newConfig.isTheSameAs(oldConfig, skipBranchCheck); } public void setOldConfig(BuildConfiguration oldConfig) { diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java index 4dd25c8b0..dcff92f61 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/pnc/PncEntitiesImporter.java @@ -84,9 +84,9 @@ public class PncEntitiesImporter { private ProductMilestone milestone; private GroupConfiguration buildGroup; private List configs; - private PigConfiguration pigConfiguration = PigContext.get().getPigConfiguration(); + private final PigConfiguration pigConfiguration = PigContext.get().getPigConfiguration(); - private PncConfigurator pncConfigurator = new PncConfigurator(); + private final PncConfigurator pncConfigurator = new PncConfigurator(); public PncEntitiesImporter() { buildConfigClient = new BuildConfigurationClient(PncClientHelper.getPncConfiguration()); @@ -97,7 +97,7 @@ public PncEntitiesImporter() { versionClient = new ProductVersionClient(PncClientHelper.getPncConfiguration()); } - public ImportResult performImport() { + public ImportResult performImport(boolean skipBranchCheck) { product = getOrGenerateProduct(); version = getOrGenerateVersion(); milestone = pncConfigurator.getOrGenerateMilestone( @@ -107,7 +107,7 @@ public ImportResult performImport() { pncConfigurator.markMilestoneCurrent(version, milestone); buildGroup = getOrGenerateBuildGroup(); - configs = getAddOrUpdateBuildConfigs(); + configs = getAddOrUpdateBuildConfigs(skipBranchCheck); log.debug("Setting up build dependencies"); setUpBuildDependencies(); @@ -221,11 +221,11 @@ private void addConfigurationToGroup(String newConfigId) { } } - private List getAddOrUpdateBuildConfigs() { + private List getAddOrUpdateBuildConfigs(boolean skipBranchCheck) { log.info("Adding/updating build configurations"); List currentConfigs = getCurrentBuildConfigs(); dropConfigsFromInvalidVersion(currentConfigs, pigConfiguration.getBuilds()); - return updateOrCreate(currentConfigs, pigConfiguration.getBuilds()); + return updateOrCreate(currentConfigs, pigConfiguration.getBuilds(), skipBranchCheck); } private Optional getBuildConfigFromName(String name) { @@ -244,14 +244,17 @@ private BuildConfiguration getBuildConfigFromId(String id) { } } - private List updateOrCreate(List currentConfigs, List builds) { + private List updateOrCreate( + List currentConfigs, + List builds, + boolean skipBranchCheck) { List buildList = new ArrayList<>(); for (BuildConfig bc : builds) { BuildConfigData data = new BuildConfigData(bc); for (BuildConfiguration config : currentConfigs) { if (config.getName().equals(bc.getName())) { data.setOldConfig(config); - if (data.shouldBeUpdated()) { + if (data.shouldBeUpdated(skipBranchCheck)) { updateBuildConfig(data, config); } } @@ -262,7 +265,7 @@ private List updateOrCreate(List currentCon if (matchedBuildConfig.isPresent()) { log.debug("Found matching build config for {}", bc.getName()); data.setOldConfig(matchedBuildConfig.get()); - if (data.shouldBeUpdated()) { + if (data.shouldBeUpdated(skipBranchCheck)) { updateBuildConfig(data, matchedBuildConfig.get()); } data.setModified(true); diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java index d5b3d8aa0..564f3cf1f 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoBuilder.java @@ -20,7 +20,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; import org.jboss.pnc.bacon.pig.impl.config.RepoGenerationData; import org.jboss.pnc.bacon.pig.impl.pnc.PncBuild; @@ -70,15 +70,16 @@ public RepoBuilder( PigConfiguration pigConfiguration, String additionalRepo, Path configurationDirectory, - Map builds) { + Map builds, + boolean removeGeneratedM2Dups) { this.pigConfiguration = pigConfiguration; this.additionalRepo = additionalRepo; this.builds = builds; this.configurationDirectory = configurationDirectory; + this.removeGeneratedM2Dups = removeGeneratedM2Dups; repoGeneration = pigConfiguration.getFlow().getRepositoryGeneration(); - removeGeneratedM2Dups = PigProperties.get().isRemoveGeneratedM2Dups(); - tempBuild = PigProperties.get().isTemporary(); + tempBuild = PigContext.get().isTempBuild(); topLevelDirectoryName = pigConfiguration.getTopLevelDirectoryPrefix() + "maven-repository"; } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoManager.java index 64c9442a5..b3950b80c 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/repo/RepoManager.java @@ -18,7 +18,7 @@ package org.jboss.pnc.bacon.pig.impl.repo; import lombok.Getter; -import org.jboss.pnc.bacon.pig.impl.PigProperties; +import org.jboss.pnc.bacon.pig.impl.PigContext; import org.jboss.pnc.bacon.pig.impl.common.DeliverableManager; import org.jboss.pnc.bacon.pig.impl.config.AdditionalArtifactsFromBuild; import org.jboss.pnc.bacon.pig.impl.config.PigConfiguration; @@ -297,7 +297,8 @@ public RepositoryData generate() { pigConfiguration, generationData.getAdditionalRepo(), configurationDirectory, - builds); + builds, + removeGeneratedM2Dups); File bomFile = new File(bomDirectory, "bom.pom"); build.downloadArtifact(generationData.getSourceArtifact(), bomFile); @@ -370,7 +371,7 @@ private void addExtraFiles(File m2Repo) { LicenseGenerator.generateLicenses( RepoDescriptor.listGavs(new File(m2Repo, RepoDescriptor.MAVEN_REPOSITORY)), new File(m2Repo, "licenses"), - PigProperties.get().isTemporary()); + PigContext.get().isTempBuild()); } } } From 8e6793dece22896cd20544ecbfbe63d372705852 Mon Sep 17 00:00:00 2001 From: Paul Gallagher Date: Tue, 9 Jun 2020 12:05:30 +0100 Subject: [PATCH 5/5] Update the mode decriptioon, added 'hasValue = false' to options that require no value like most of the skip options, added a deprecated check in javadoc for yaml usage of customPmeParameters everyone should use alignmentParameters --- .../main/java/org/jboss/pnc/bacon/pig/Pig.java | 16 +++++++++++++++- .../pig/impl/config/JavadocGenerationData.java | 15 +++++++++++++++ .../bacon/pig/impl/javadoc/JavadocManager.java | 4 ++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java index a122b6639..19150dda5 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/Pig.java @@ -47,7 +47,7 @@ Pig.TriggerAddOns.class }) public class Pig extends AbstractCommand { - public static final String REBUILD_MODE_DESC = "If specified, artifacts from temporary builds will have timestamp in versions"; + public static final String REBUILD_MODE_DESC = "The build mode EXPLICIT_DEPENDENCY_CHECK, IMPLICIT_DEPENDENCY_CHECK, FORCE. Defaults to EXPLICIT"; public static final String REBUILD_MODE_DEFAULT = "EXPLICIT_DEPENDENCY_CHECK"; public static final String REBUILD_MODE = "mode"; public static final String TEMP_BUILD_TIME_STAMP = "tempBuildTimeStamp"; @@ -74,6 +74,7 @@ public abstract class PigCommand extends AbstractCommand { @Option( shortName = TEMP_BUILD_SHORT, name = TEMP_BUILD, + hasValue = false, overrideRequired = true, defaultValue = TEMP_BUILD_DEFAULT, description = TEMP_BUILD_DESC) @@ -112,6 +113,7 @@ public class Run extends PigCommand { @Option( name = TEMP_BUILD_TIME_STAMP, overrideRequired = true, + hasValue = false, defaultValue = TEMP_BUILD_TIME_STAMP_DEFAULT, description = TEMP_BUILD_TIME_STAMP_DESC) private boolean tempBuildTS; @@ -126,6 +128,7 @@ public class Run extends PigCommand { @Option( name = "skipRepo", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip maven repository generation") private boolean skipRepo; @@ -133,6 +136,7 @@ public class Run extends PigCommand { @Option( name = "skipPncUpdate", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip updating PNC entities. Use only if you have all entities created properly.") private boolean skipPncUpdate; @@ -140,6 +144,7 @@ public class Run extends PigCommand { @Option( name = "skipBuilds", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip PNC builds. Use when all your builds went fine, something failed later " + "and you want to retry generating deliverables without rebuilding.") @@ -148,6 +153,7 @@ public class Run extends PigCommand { @Option( name = "skipSources", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip sources generation.") private boolean skipSources; @@ -155,6 +161,7 @@ public class Run extends PigCommand { @Option( name = "skipJavadoc", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip Javadoc generation.") private boolean skipJavadoc; @@ -162,6 +169,7 @@ public class Run extends PigCommand { @Option( name = "skipLicenses", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip Licenses generation.") private boolean skipLicenses; @@ -169,6 +177,7 @@ public class Run extends PigCommand { @Option( name = "skipSharedContent", overrideRequired = true, + hasValue = false, defaultValue = "false", description = "Skip generating shared content request input.") private boolean skipSharedContent; @@ -176,6 +185,7 @@ public class Run extends PigCommand { @Option( name = REMOVE_M2_DUPLICATES, overrideRequired = true, + hasValue = false, defaultValue = REMOVE_M2_DUPLICATES_DEFAULT, description = REMOVE_M2_DUPLICATES_DESC) private boolean removeGeneratedM2Dups; @@ -183,6 +193,7 @@ public class Run extends PigCommand { @Option( name = SKIP_BRANCH_CHECK, overrideRequired = true, + hasValue = false, defaultValue = SKIP_BRANCH_CHECK_DEFAULT, description = SKIP_BRANCH_CHECK_DESC) private boolean skipBranchCheck; @@ -221,6 +232,7 @@ public class Configure extends PigCommand { @Option( name = SKIP_BRANCH_CHECK, overrideRequired = true, + hasValue = false, defaultValue = SKIP_BRANCH_CHECK_DEFAULT, description = SKIP_BRANCH_CHECK_DESC) private boolean skipBranchCheck; @@ -245,6 +257,7 @@ public class Build extends PigCommand> { @Option( name = TEMP_BUILD_TIME_STAMP, overrideRequired = true, + hasValue = false, defaultValue = TEMP_BUILD_TIME_STAMP_DEFAULT, description = TEMP_BUILD_TIME_STAMP_DESC) private boolean tempBuildTS; @@ -270,6 +283,7 @@ public class GenerateRepository extends PigCommand { @Option( name = REMOVE_M2_DUPLICATES, overrideRequired = true, + hasValue = false, defaultValue = REMOVE_M2_DUPLICATES_DEFAULT, description = REMOVE_M2_DUPLICATES_DESC) private boolean removeGeneratedM2Dups; diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java index 772d31c88..25feb223b 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/config/JavadocGenerationData.java @@ -24,16 +24,31 @@ import java.util.Set; import java.util.TreeSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @author Michal Szynkiewicz, michal.l.szynkiewicz@gmail.com
* Date: 5/25/18 */ @Data public class JavadocGenerationData extends GenerationData { + private static final Logger log = LoggerFactory.getLogger(JavadocGenerationData.class); + private List sourceBuilds = new ArrayList<>(); private String scmRevision; private String generationProject; private String buildScript; + // customPmeParameters Deprecated - alignmentParameters should be used private Set customPmeParameters = new TreeSet<>(); + private Set alignmentParameters = new TreeSet<>(); private String importBom; + + public Set getAlignmentParameters() { + if (!customPmeParameters.isEmpty() && alignmentParameters.isEmpty()) { + log.warn("[Deprecated] Please rename 'customPmeParameters' section to 'alignmentParameters'"); + alignmentParameters = customPmeParameters; + } + return alignmentParameters; + } } diff --git a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java index 17e8c7971..adcdf9428 100644 --- a/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java +++ b/pig/src/main/java/org/jboss/pnc/bacon/pig/impl/javadoc/JavadocManager.java @@ -276,8 +276,8 @@ private boolean runPME() { "-f " + topLevelDirectory.getPath() + File.separator + "pom.xml" + " -DprofileInjection=" + project_gid + ":" + project_aid + ":" + project_version + " -s " + settingsXml + " -Dmaven.repo.local=" + localRepo + " -t"); - if (generationData.getCustomPmeParameters() != null && !generationData.getCustomPmeParameters().isEmpty()) { - for (String parameter : generationData.getCustomPmeParameters()) { + if (generationData.getAlignmentParameters() != null && !generationData.getAlignmentParameters().isEmpty()) { + for (String parameter : generationData.getAlignmentParameters()) { cmd.append(" " + parameter); } }