Task/585 reorg

.github/workflows/build.yml

@@ -42,10 +42,10 @@ jobs:
         run: ./gradlew build
       - name: Integration Tests CWMS
         if: always()
-        run: ./gradlew integrationTestCWMS --stacktrace --info
+        run: ./gradlew :testing:rest-api:testCWMSOracle --stacktrace --info
       - name: Integration Tests OpenTSDB
         if: always()
-        run: ./gradlew integrationTestOpenTSDB --stacktrace --info
+        run: ./gradlew :testing:rest-api:testOpenDCSPostgres --stacktrace --info
       - name: Upload WAR files
         uses: actions/upload-artifact@v5.0.0
         with:

.github/workflows/owasp_zap.yml

@@ -25,17 +25,17 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
           restore-keys: ${{ runner.os }}-gradle
       - name: Run OWASP Zap
-        run: ./gradlew :opendcs-integration-test:owaspZap --stacktrace
+        run: ./gradlew :testing:rest-api:owaspZap --stacktrace
       - name: Upload Full HTML Report
         uses: actions/upload-artifact@v5.0.0
         with:
           name: zap_report.html
-          path: opendcs-integration-test/build/test-results/owasp_zap/zap_report.html
+          path: integrationtesting/rest-api/build/test-results/owasp_zap/zap_report.html
           retention-days: 1
           if-no-files-found: error
       - name: Trim Summary Report
         run: |
-          sudo sed '/## Alert Detail/Q' opendcs-integration-test/build/test-results/owasp_zap/zap_report.md > ./zap_report_summary.md
+          sudo sed '/## Alert Detail/Q' integrationtesting/rest-api/build/test-results/owasp_zap/zap_report.md > ./zap_report_summary.md
       - name: Generate GitHub Job Summary
         uses: x-color/github-actions-job-summary@v0.1.1
         with:

.github/workflows/publish.yml

@@ -22,17 +22,17 @@ jobs:
         run: ./gradlew generateOpenAPI
       - name: Attach OpenDCS REST API WAR file
         run: |
-          gh release upload ${{ github.event.release.tag_name }} opendcs-rest-api/build/libs/opendcs-rest-api-${{ github.event.release.tag_name }}.war --repo ${{ github.repository }}
+          gh release upload ${{ github.event.release.tag_name }} java/opendcs-rest-api/build/libs/opendcs-rest-api-${{ github.event.release.tag_name }}.war --repo ${{ github.repository }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Attach OpenDCS Web Client WAR file
+      - name: Attach OpenDCS Web UI WAR file
         run: |
-          gh release upload ${{ github.event.release.tag_name }} opendcs-web-client/build/libs/opendcs-web-client-${{ github.event.release.tag_name }}.war --repo ${{ github.repository }}
+          gh release upload ${{ github.event.release.tag_name }} java/opendcs-web-ui/build/libs/opendcs-web-ui-${{ github.event.release.tag_name }}.war --repo ${{ github.repository }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Attach OpenAPI Specification
         run: |
-            gh release upload ${{ github.event.release.tag_name }} opendcs-rest-api/build/swagger/opendcs-openapi.json --repo ${{ github.repository }}
+            gh release upload ${{ github.event.release.tag_name }} java/opendcs-rest-api/build/swagger/opendcs-openapi.json --repo ${{ github.repository }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   container-release:

Dockerfile

@@ -27,8 +27,8 @@ CMD ["/usr/local/tomcat/bin/catalina.sh","run"]
 
 FROM tomcat_base AS api
 
-COPY --from=builder /builddir/opendcs-rest-api/build/libs/*.war /usr/local/tomcat/webapps/odcsapi.war
-COPY --from=builder /builddir/opendcs-web-client/build/libs/*.war /usr/local/tomcat/webapps/ROOT.war
+COPY --from=builder /builddir/java/opendcs-rest-api/build/libs/*.war /usr/local/tomcat/webapps/odcsapi.war
+COPY --from=builder /builddir/java/opendcs-web-ui/build/libs/*.war /usr/local/tomcat/webapps/ROOT.war
 COPY /docker_files/tomcat/conf/context.xml /usr/local/tomcat/conf/Catalina/localhost/odcsapi.xml
 COPY /docker_files/tomcat/conf/tomcat-server.xml /usr/local/tomcat/conf/server.xml
 COPY /docker_files/tomcat/conf/setenv.sh /usr/local/tomcat/bin

README.md

@@ -9,8 +9,8 @@ OpenDCS Rest API is web application that provides access to the OpenDCS database
 OpenDCS Rest API is intended to run as a stand-alone Java program. It uses embedded JETTY to implement the web services.
 
 # Structure
-./opendcs-rest-api - contains source files for the OpenDCS REST API
-./opendcs-web-client - contains source files for the OpenDCS Web Application Client
+./java/opendcs-rest-api - contains source files for the OpenDCS REST API
+./java/opendcs-web-ui - contains source files for the OpenDCS Web Application Client
 ./opendcs-integration-test - contains scripts for running embedded tomcat to deploy the REST API and Web Client wars for testing.
 
 
@@ -36,7 +36,7 @@ properties that should be configured for your system.
 - `opendcs.rest.api.authorization.jwt.issuer.url`  - for openid authorization this is the Issuer URL
 
 ## OPENDCS Web Client
-The gradle task `./gradlew :opendcs-web-client:war` will create a war file in the `build/libs` directory.
+The gradle task `./gradlew :opendcs-web-ui:war` will create a war file in the `build/libs` directory.
 
 # CI/CD
 The GitHub Action workflow [default.yml](./.github/workflows/default.yml) contains the primary CI/CD pipeline for the project.

opendcs-integration-test/README.md → integrationtesting/rest-api/README.md

@@ -21,5 +21,5 @@ environment variables for the database connection information and CWMS office id
 REST API config files can be found at [src/test/resources/rest-api/conf](src/test/resources/rest-api/conf)
 which includes separate configurations for CWMS and OpenTSDB.
 
-Web Client config files can be found at [src/test/resources/web-client/conf](src/test/resources/web-client/conf)
+Web Client config files can be found at [src/test/resources/web-ui/conf](src/test/resources/web-ui/conf)
 which includes separate configurations for CWMS and OpenTSDB.

opendcs-integration-test/build.gradle → integrationtesting/rest-api/build.gradle

@@ -73,7 +73,7 @@ configurations.all {
 }
 
 
-tasks.register('generateConfig', Copy) {
+def genConfig = tasks.register('generateConfig', Copy) {
     doFirst {
         project.delete layout.buildDirectory.file("tomcat/")
         project.mkdir layout.buildDirectory.file("tomcat/logs")
@@ -88,10 +88,10 @@ tasks.register('generateConfig', Copy) {
 tasks.register('run', JavaExec) {
     dependsOn build
     def restWar = project(':opendcs-rest-api').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    def guiWar = project(':opendcs-web-client').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
+    def guiWar = project(':opendcs-web-ui').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
     inputs.files(restWar,guiWar)
     group "application"
-    dependsOn(':opendcs-rest-api:war', ':opendcs-web-client:war')
+    dependsOn(':opendcs-rest-api:war', ':opendcs-web-ui:war')
     dependsOn generateConfig
     dependsOn downloadAndUnzipInstaller
     classpath = sourceSets.main.runtimeClasspath
@@ -116,9 +116,9 @@ tasks.register('run', JavaExec) {
 tasks.register('owaspZap', JavaExec) {
     dependsOn build
     def restWar = project(':opendcs-rest-api').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    def guiWar = project(':opendcs-web-client').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
+    def guiWar = project(':opendcs-web-ui').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
     group "application"
-    dependsOn(':opendcs-rest-api:war', ':opendcs-web-client:war')
+    dependsOn(':opendcs-rest-api:war', ':opendcs-web-ui:war')
     dependsOn generateConfig
     dependsOn downloadAndUnzipInstaller
 
@@ -140,8 +140,8 @@ tasks.register('owaspZap', JavaExec) {
     systemProperties.put("DCSTOOL_HOME", outputDir)
 }
 
-
-tasks.register('downloadAndUnzipInstaller') {
+// to be removed during merge into main code
+def odcsInstall = tasks.register('downloadAndUnzipInstaller') {
     def zipUrl = "https://github.com/opendcs/opendcs/releases/download/${libs.versions.opendcs.get()}/opendcs-${libs.versions.opendcs.get()}.zip"
     def outputDir = project.layout.buildDirectory.dir("opendcs-install").get()
     def zipFile = project.layout.buildDirectory.dir("download/opendcs-install.zip").get().getAsFile()
@@ -163,101 +163,101 @@ tasks.register('downloadAndUnzipInstaller') {
     }
 }
 
-tasks.register('integrationTestOpenTSDB', Test) {
-    group "verification"
-    useJUnitPlatform {
-        includeTags "integration"
-        includeTags "integration-opentsdb-only"
-    }
-    // with gradle 9.2 it seems we need to set these manually
-    testClassesDirs = sourceSets.test.output.classesDirs
-    classpath = sourceSets.test.runtimeClasspath
-    shouldRunAfter test
-    dependsOn(':opendcs-rest-api:war')
-    dependsOn(':opendcs-web-client:war')
-    dependsOn generateConfig
-    dependsOn downloadAndUnzipInstaller
-    jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource"
-    jvmArgs += "-Dcatalina.base=${layout.buildDirectory.file("tomcat")}"
-    jvmArgs += "-Dopendcs.test.integration.db=OpenDCS-Postgres"
-    def restWar = project(':opendcs-rest-api').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    def guiWar = project(':opendcs-web-client').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    jvmArgs += "-Dopendcs.restapi.warfile=${restWar}"
-    jvmArgs += "-Dopendcs.gui.warfile=${guiWar}"
-    def outputDir = downloadAndUnzipInstaller.outputs.files.singleFile
-    systemProperties.put("DCSTOOL_HOME", outputDir)
-    systemProperties.put("DCSTOOL_USERDIR", "build/resources/test/tomcat/conf")
-    systemProperties.put("java.util.logging.config.file", "build/tomcat/conf/logging.properties")
-    finalizedBy integrationTestReportOpenTSDB
-}
+def commonJacocoExcludes = [
+  "**/easy_install/**",
+  "**/python_packages",
+  "**/certifi/**",
+  "**/chardet/**",
+  "**/urllib3/**",
+  "**/requests/**",
+  "**/idna/**",
+  "**/pkg_resources/**"
+]
 
-tasks.register('integrationTestCWMS', Test) {
-    group "verification"
-    useJUnitPlatform {
-        includeTags "integration"
-        includeTags "integration-cwms-only"
+// do need to add OpenDCS-Oracle support eventually.
+def testEngines = (project.findProperty("opendcs.test.engine")?: "OpenDCS-Postgres,CWMS-Oracle").split(",")
+
+testEngines.each { engine  ->
+
+    def tmpdir = "${project.layout.buildDirectory.get()}/runs/${engine}/tmp"
+
+    def task = tasks.register("test"+engine.replace("-",""), Test) { task ->
+        group "verification"
+        dependsOn odcsInstall
+        dependsOn(':opendcs-rest-api:war')
+        dependsOn(':opendcs-web-ui:war')
+        dependsOn genConfig
+        useJUnitPlatform()
+
+        // with gradle 9.2 it seems we need to set these manually
+        testClassesDirs = sourceSets.test.output.classesDirs
+        classpath = sourceSets.test.runtimeClasspath
+
+        enabled = gradle.startParameter.taskNames.contains(":testing:" + project.name + ":test") ||
+                  gradle.startParameter.taskNames.contains(":testing:" + project.name + ":" +task.name)
+        doFirst {
+            project.mkdir(tmpdir)
+        }
+
+        inputs.property "opendcs.test.engine", engine
+
+        systemProperties += project.properties.findAll {k, v -> k.startsWith("opendcs")}
+        systemProperties += project.properties.findAll {k, v -> k.startsWith("testcontainer")}
+        systemProperties += ["opendcs.test.engine": engine]
+
+        jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource"
+        jvmArgs += "-Dcatalina.base=${layout.buildDirectory.file("tomcat")}"
+        jvmArgs += "-Dopendcs.test.integration.db=${engine}"
+        def restWar = project(':opendcs-rest-api').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
+        def guiWar = project(':opendcs-web-ui').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
+        jvmArgs += "-Dopendcs.restapi.warfile=${restWar}"
+        jvmArgs += "-Dopendcs.gui.warfile=${guiWar}"
+        // not needed for the Postgres setup but it doesn't get in the way and we need to make some tweaks to the fixture
+        // that we can't make in this PR.
+        jvmArgs += "-Dtestcontainer.cwms.bypass.cwms.pass=dcs_user"
+        def odcsInstallDir = downloadAndUnzipInstaller.outputs.files.singleFile
+        systemProperties.put("DCSTOOL_HOME", odcsInstallDir)
+        systemProperties.put("DCSTOOL_USERDIR", "build/resources/test/tomcat/conf")
+        systemProperties.put("java.util.logging.config.file", "build/tomcat/conf/logging.properties")
+        reports {
+            html {}
+            junitXml {}
+        }
+        jacoco {
+            excludes += commonJacocoExcludes
+        }
     }
-    // with gradle 9.2 it seems we need to set these manually
-    testClassesDirs = sourceSets.test.output.classesDirs
-    classpath = sourceSets.test.runtimeClasspath
-    shouldRunAfter test
-    dependsOn(':opendcs-rest-api:war')
-    dependsOn(':opendcs-web-client:war')
-    dependsOn generateConfig
-    dependsOn downloadAndUnzipInstaller
-    jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource"
-    jvmArgs += "-Dcatalina.base=${layout.buildDirectory.file("tomcat")}"
-    jvmArgs += "-Dopendcs.test.integration.db=CWMS-Oracle"
-    jvmArgs += "-Dtestcontainer.cwms.bypass.cwms.pass=dcs_user"
-    def restWar = project(':opendcs-rest-api').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    def guiWar = project(':opendcs-web-client').tasks.named('war').get().archiveFile.get().asFile.getAbsolutePath()
-    jvmArgs += "-Dopendcs.restapi.warfile=${restWar}"
-    jvmArgs += "-Dopendcs.gui.warfile=${guiWar}"
-    def outputDir = downloadAndUnzipInstaller.outputs.files.singleFile
-    systemProperties.put("DCSTOOL_HOME", outputDir)
-    systemProperties.put("DCSTOOL_USERDIR", "build/resources/test/tomcat/conf")
-    systemProperties.put("java.util.logging.config.file", "build/tomcat/conf/logging.properties")
-    finalizedBy integrationTestReportCWMS
-}
+    test.dependsOn task
 
+    def jtr = tasks.register("jacocoTestReport"+engine.replace("-",""), JacocoReport) {
+        dependsOn task
+        sourceSets project(":opendcs-rest-api").sourceSets.main
+        executionData = files("${buildDir}/jacoco/test"+engine.replace("-","")+".exec") // Use the .exec file from the custom test task
+        reports {
+            html {
+                outputLocation = jacoco.reportsDirectory.dir("jacoco-${engine}-html")
+            }
+            xml {
+                outputLocation = jacoco.reportsDirectory.file("jacoco-${engine}.xml")
+            }
+
+        }
+
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it, exclude: commonJacocoExcludes)
+        }))
 
-tasks.register('integrationTestReportCWMS', JacocoReport) {
-    group = "verification"
-    dependsOn tasks.named('integrationTestCWMS')
-    additionalSourceDirs.setFrom(files(project(':opendcs-rest-api').sourceSets.main.allSource.srcDirs))
-    executionData = files(
-            layout.buildDirectory.dir("jacoco/integrationTestCWMS.exec")
-    )
-    classDirectories.setFrom(files(
-            project(':opendcs-rest-api').sourceSets.main.output,
-            sourceSets.main.output
-    ))
-    reports {
-        xml.required.set(true)
     }
-}
 
-tasks.register('integrationTestReportOpenTSDB', JacocoReport) {
-    group = "verification"
-    dependsOn tasks.named('integrationTestOpenTSDB')
-    additionalSourceDirs.setFrom(files(project(':opendcs-rest-api').sourceSets.main.allSource.srcDirs))
-    executionData = files(
-            layout.buildDirectory.dir("jacoco/integrationTestOpenTSDB.exec")
-    )
-    classDirectories.setFrom(files(
-            project(':opendcs-rest-api').sourceSets.main.output,
-            sourceSets.main.output
-    ))
-    reports {
-        xml.required.set(true)
+    task.configure {
+        finalizedBy jtr
     }
+
 }
 
 test {
     useJUnitPlatform {
-        excludeTags "integration"
-        excludeTags "integration-opentsdb-only"
-        excludeTags "integration-cwms-only"
+    exclude '**/*'
     }
 }
 

opendcs-integration-test/src/main/java/org/opendcs/odcsapi/fixtures/TomcatServer.java → integrationtesting/rest-api/src/main/java/org/opendcs/odcsapi/fixtures/TomcatServer.java

@@ -196,8 +196,8 @@ private static void setupDbForBypass(DbType dbType)
 			System.setProperty(DB_DATASOURCE_CLASS, DataSourceFactory.class.getName());
 			System.setProperty(DB_VALIDATION_QUERY, "SELECT 1 FROM dual");
 			System.setProperty(DB_URL, System.getProperty("testcontainer.cwms.bypass.url"));
-			System.setProperty(DB_USERNAME, System.getProperty("testcontainer.cwms.bypass.office.eroc") + "WEBTEST");
-			System.setProperty(DB_PASSWORD, System.getProperty("testcontainer.cwms.bypass.cwms.pass"));
+			System.setProperty(DB_USERNAME, System.getProperty("testcontainer.cwms.bypass.dcs.user", "dcs_user"));
+			System.setProperty(DB_PASSWORD, System.getProperty("testcontainer.cwms.bypass.dcs.pass", "dcs_user"));
 		}
 		else
 		{
@@ -223,11 +223,10 @@ private static ConfigurationProvider getProvider(DbType dbType)
 		Iterator<ConfigurationProvider> configs = loader.iterator();
 
 		ConfigurationProvider configProvider = null;
-		System.out.println("DbType " + dbType.name());
+		log.info("DbType {}", dbType.name());
 		while(configs.hasNext())
 		{
 			ConfigurationProvider configProviderTmp = configs.next();
-			System.out.println("Current provider: " + configProviderTmp.getImplementation());
 			if(configProviderTmp.getImplementation().equals(dbType.getProvider()))
 			{
 				configProvider = configProviderTmp;

opendcs-integration-test/src/test/java/org/opendcs/odcsapi/res/it/AlgorithmResourcesIT.java → integrationtesting/rest-api/src/test/java/org/opendcs/odcsapi/res/it/AlgorithmResourcesIT.java

@@ -20,19 +20,14 @@
 
 import io.restassured.filter.log.LogDetail;
 import io.restassured.filter.session.SessionFilter;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.opendcs.odcsapi.beans.ApiAlgorithm;
-import org.opendcs.odcsapi.fixtures.DatabaseContextProvider;
 
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 
-@Tag("integration")
-@ExtendWith(DatabaseContextProvider.class)
 final class AlgorithmResourcesIT extends BaseIT
 {
 	@BeforeEach

opendcs-integration-test/src/test/java/org/opendcs/odcsapi/res/it/AppResourcesIT.java → integrationtesting/rest-api/src/test/java/org/opendcs/odcsapi/res/it/AppResourcesIT.java

@@ -33,11 +33,8 @@
 import opendcs.dai.LoadingAppDAI;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.opendcs.odcsapi.beans.ApiAppStatus;
-import org.opendcs.odcsapi.fixtures.DatabaseContextProvider;
 
 import static io.restassured.RestAssured.given;
 import static org.hamcrest.Matchers.equalTo;
@@ -47,8 +44,6 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-@Tag("integration")
-@ExtendWith(DatabaseContextProvider.class)
 final class AppResourcesIT extends BaseIT
 {
 	private static Long appid = null;