Optionally create time series

CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Offer possibility to choose, if time series shall be created or no
+
+### Changed
+- Actor-based implementation for better concurrency
 
 ## [1.0.0] - 2021-08-03
 ### Added

README.md

@@ -11,3 +11,19 @@ SimBench has been research project of Kassel university, Fraunhofer IEE, RWTH Aa
 The data is availabe at https://simbench.de.
 
 There is another conversion tool available [here](https://github.com/e2nIEE/simbench), that makes the models available to the well known [pandapower](https://github.com/e2nIEE/pandapower) simulation software.
+
+## Hints for usage
+If you intend to convert bigger models, like `1-MVLV-rural-all-...`, it may consume a considerable amount of resources.
+Therefore, you have two options to make conversion more convenient.
+If you don't need time resolved grid usage patterns, you can turn off their conversion by `conversion.createTimeSeries = true` within the config.
+
+If you are interested in obtaining the time series, it is recommended to set some JVM options.
+Of course, adapting the heap size is advisory.
+For above-mentioned grid, `-Xmx20g` was sufficient, if the config was chosen as follows.
+Of course this will also be dependend on your system.
+```hocon
+conversion.participantWorkersPerType = 5
+io.output.workers = 30
+```
+If running the conversion with Java 8, it is also advisory to define a specific garbage collector to use.
+The G1 garbage collector `-XX:+UseG1GC` was a good choice for above-mentioned example application.

build.gradle

@@ -1,4 +1,7 @@
+import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer
+
 plugins {
+	id 'application'
 	id "com.jfrog.artifactory" version "4.24.14"
 	id 'groovy' // groovy support
 	id 'java' // java support
@@ -7,6 +10,7 @@ plugins {
 	id 'com.diffplug.spotless' version '5.14.2'// code format
 	id "de.undercouch.download" version "4.1.2" // downloads plugin
 	id 'jacoco' // java code coverage plugin
+	id "com.github.johnrengelman.shadow" version "7.0.0" // fat jar
 	id "org.sonarqube" version "3.3" // sonarqube
 	id 'org.scoverage' version '6.0.0' // Code coverage plugin for scala
 	id "com.github.maiflai.scalatest" version "0.31" // run scalatest without specific task
@@ -18,7 +22,7 @@ ext {
 
 	scalaVersion = '2.13'
 	scalaBinaryVersion = '2.13.6'
-	tscfgVersion = '0.9.986'
+	tscfgVersion = '0.9.993'
 	slf4jVersion = '1.7.32'
 
 	scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator //location of script plugins
@@ -36,6 +40,8 @@ apply from: scriptsLocation + 'checkJavaVersion.gradle'
 apply from: scriptsLocation + 'tscfg.gradle' // config tasks
 apply from: scriptsLocation + 'scoverage.gradle'
 
+application.mainClassName = 'edu.ie3.simbench.main.RunSimbench'
+
 repositories {
 	mavenCentral()
 	maven { url 'https://www.jitpack.io' } // allows github repos as dependencies
@@ -64,11 +70,11 @@ dependencies {
 	implementation 'tech.units:indriya:2.1.2'
 
 	// logging
-	implementation 'org.apache.logging.log4j:log4j-api:+' // log4j
-	implementation 'org.apache.logging.log4j:log4j-core:+' // log4j
-	implementation 'org.apache.logging.log4j:log4j-slf4j-impl:+' // log4j -> slf4j
+	implementation 'org.apache.logging.log4j:log4j-api:2.14.1' // log4j
+	implementation 'org.apache.logging.log4j:log4j-core:2.14.1' // log4j
+	implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1' // log4j -> slf4j
 
-	implementation "com.typesafe.scala-logging:scala-logging_${scalaVersion}:+" // akka scala logging
+	implementation "com.typesafe.scala-logging:scala-logging_${scalaVersion}:3.9.4" // akka scala logging
 	implementation "org.slf4j:log4j-over-slf4j:${slf4jVersion}" // slf4j -> log4j
 
 	// NEW scala libs //
@@ -82,21 +88,43 @@ dependencies {
 	testImplementation "org.pegdown:pegdown:1.6.0" // HTML report for scalatest
 	implementation 'org.mockito:mockito-core:3.11.2' // mocking framework
 
+	// akka actor system //
+	implementation platform("com.typesafe.akka:akka-bom_${scalaVersion}:2.6.14")
+	implementation "com.typesafe.akka:akka-actor-typed_${scalaVersion}"
+	implementation "com.typesafe.akka:akka-stream_${scalaVersion}"
+	testImplementation "com.typesafe.akka:akka-actor-testkit-typed_${scalaVersion}"
+
 	// config //
-	implementation 'com.typesafe:config:+'
+	implementation 'com.typesafe:config:1.4.1'
 	implementation "com.github.carueda:tscfg_2.13:${tscfgVersion}"
 
 	// cmd args parser //
 	implementation "com.github.scopt:scopt_${scalaVersion}:+"
 
 	/* Handling compressed archives */
-	implementation "org.apache.commons:commons-compress:+"
+	implementation "org.apache.commons:commons-compress:1.21"
 }
 
 wrapper {
-	gradleVersion = '7.1.1'
+	gradleVersion = '7.2'
 }
 
 tasks.withType(JavaCompile) {
 	options.encoding = 'UTF-8'
 }
+
+tasks.withType(Javadoc){
+	options.encoding = 'UTF-8'
+}
+
+//////////////////////////////////////////////////////////////////////
+// Build akka'able fat jar using the gradle shadow plugin
+// see http://www.sureshpw.com/2015/10/building-akka-bundle-with-all.html
+// and https://github.com/akka/akka/issues/24248
+//////////////////////////////////////////////////////////////////////
+shadowJar {
+	transform(AppendingTransformer) {
+		resource = 'reference.conf'
+	}
+	zip64 = true
+}

docs/diagrams/protocol/protocol.puml

@@ -0,0 +1,56 @@
+@startuml
+
+participant Main
+participant Coordinator
+participant Converter
+participant GridConverter
+participant ResConverter
+collections ResConverterWorker
+participant Mutator
+
+Main -> Coordinator: Start(Config)
+== Init ==
+Coordinator --[#blue]> Converter: <font color=blue>spawn & watch</font>
+Coordinator -> Converter: Init(...)
+Converter --[#blue]> Mutator: <font color=blue>spawn & watch</font>
+Converter -> Mutator: Init(Config)
+Converter <- Mutator: Ready
+Coordinator <- Converter: ConverterInitialized
+Coordinator -> Converter: Convert(String)
+
+== Converting grid structure ==
+Converter --[#blue]> GridConverter: <font color=blue>spawn & watch</font>
+Converter -> GridConverter: ConvertGridStructure(...)
+Converter <- GridConverter: GridStructureConverted(...)
+note left: Now all other entities\ncan be converted (not)\nconsidering islanded\nnodes
+
+== Converting participants ==
+Converter --[#blue]> ResConverter: <font color=blue>spawn & watch</font>
+Converter -> ResConverter: Init(...)
+ResConverter --[#blue]> ResConverterWorker:  <font color=blue>spawn & watch</font>
+Converter <- ResConverter: ResConverterReady
+Converter -> ResConverter: Convert(...)
+ResConverter -> ResConverterWorker: Convert(...)
+ResConverterWorker -> Mutator: PersistTimeSeries(...)
+activate Mutator
+ResConverterWorker <- Mutator: TimeSeriesPersisted(...)
+ResConverter <- ResConverterWorker: Converted(...)
+ResConverter --[#blue]> ResConverterWorker: <font color=blue>terminate</font>
+ResConverter <--[#blue] ResConverterWorker
+Converter <- ResConverter: ResConverted(...)
+
+== Mutate Grid structure ==
+Converter -> Mutator: PersistGridStructure(...)
+Converter -> Mutator: PersistTimeSeriesMapping(...)
+Converter -> Mutator: PersistNodalResults(...)
+Converter <- Mutator: GridStructurePersisted(...)
+Converter <- Mutator: TimeSeriesMappingPersisted(...)
+Converter <- Mutator: NodalResultsPersisted(...)
+
+== Shutdown ==
+Converter -> Mutator: Shutdown
+Converter <- Mutator: Done
+deactivate Mutator
+
+Coordinator <- Converter: Converted(String)
+@enduml

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -72,7 +72,7 @@ case "`uname`" in
   Darwin* )
     darwin=true
     ;;
-  MINGW* )
+  MSYS* | MINGW* )
     msys=true
     ;;
   NONSTOP* )
@@ -130,7 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath

gradlew.bat

@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -54,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -64,29 +64,14 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

inputData/config/allNoSwitches.conf

@@ -1,3 +1,5 @@
+include "common/akka.conf"
+
 # Simple config to convert all extra high voltage grids
 io {
   input {
@@ -19,6 +21,7 @@ io {
 
     targetFolder = "convertedData/multiVoltLvlNoSwitches"
     compress = true
+    workers = 20
   }
 
   simbenchCodes = [
@@ -150,3 +153,9 @@ io {
     "1-MV-urban--2-no_sw"
   ]
 }
+
+conversion {
+  removeSwitches = false
+  participantWorkersPerType = 20
+  createTimeSeries = true
+}

inputData/config/common/akka.conf

@@ -0,0 +1,10 @@
+restricted-core-pool-dispatcher {
+  type = Dispatcher
+  executor = "thread-pool-executor"
+  thread-pool-executor {
+    core-pool-size-min = 2
+    core-pool-size-factor = 2.0
+    core-pool-size-max = 10
+  }
+  throughput = 1
+}

inputData/config/multiVoltLvlNoSwitches.conf

@@ -1,3 +1,5 @@
+include "common/akka.conf"
+
 # Simple config to convert all extra high voltage grids
 io {
   input {
@@ -19,6 +21,7 @@ io {
 
     targetFolder = "convertedData/multiVoltLvlNoSwitches"
     compress = true
+    workers = 20
   }
 
   simbenchCodes = [
@@ -112,3 +115,9 @@ io {
     "1-MVLV-urban-all-2-no_sw"
   ]
 }
+
+conversion {
+  removeSwitches = false
+  participantWorkersPerType = 20
+  createTimeSeries = true
+}

inputData/config/mvLvNoSwitches.conf

@@ -1,3 +1,6 @@
+include "common/akka.conf"
+restricted-core-pool-dispatcher.thread-pool-executor.core-pool-size-max = 8
+
 # Simple config to convert all models that comprise mv and lv -- either single or connected
 io {
   input {
@@ -19,6 +22,7 @@ io {
 
     targetFolder = "convertedData/mvLvNoSwitches"
     compress = true
+    workers = 30
   }
 
   simbenchCodes = [
@@ -76,7 +80,7 @@ io {
     "1-LV-urban6--0-no_sw",
     "1-LV-urban6--1-no_sw",
     "1-LV-urban6--2-no_sw",
-//    "1-MV-comm--0-no_sw",
+    "1-MV-comm--0-no_sw",
     "1-MV-comm--1-no_sw",
     "1-MV-comm--2-no_sw",
     "1-MV-rural--0-no_sw",
@@ -89,21 +93,23 @@ io {
     "1-MV-urban--1-no_sw",
     "1-MV-urban--2-no_sw",
     # Causing memory issues
-    #"1-MVLV-comm-all-0-no_sw",
-    #"1-MVLV-comm-all-1-no_sw",
-    #"1-MVLV-comm-all-2-no_sw",
-    #"1-MVLV-rural-all-0-no_sw",
-    #"1-MVLV-rural-all-1-no_sw",
-    #"1-MVLV-rural-all-2-no_sw",
-    #"1-MVLV-semiurb-all-0-no_sw",
-    #"1-MVLV-semiurb-all-1-no_sw",
-    #"1-MVLV-semiurb-all-2-no_sw",
-    #"1-MVLV-urban-all-0-no_sw",
-    #"1-MVLV-urban-all-1-no_sw",
-    #"1-MVLV-urban-all-2-no_sw",
+    "1-MVLV-comm-all-0-no_sw",
+    "1-MVLV-comm-all-1-no_sw",
+    "1-MVLV-comm-all-2-no_sw",
+    "1-MVLV-rural-all-0-no_sw",
+    "1-MVLV-rural-all-1-no_sw",
+    "1-MVLV-rural-all-2-no_sw",
+    "1-MVLV-semiurb-all-0-no_sw",
+    "1-MVLV-semiurb-all-1-no_sw",
+    "1-MVLV-semiurb-all-2-no_sw",
+    "1-MVLV-urban-all-0-no_sw",
+    "1-MVLV-urban-all-1-no_sw",
+    "1-MVLV-urban-all-2-no_sw"
   ]
 }
 
 conversion {
   removeSwitches = true
+  participantWorkersPerType = 5
+  createTimeSeries = true
 }