lst = new LinkedList<>();
String urls = _options.get(urlOptionName);
- if(urls == null) return null;
+ if (urls == null) return null;
JSONArray arr = new JSONArray(urls);
- for(int i = 0; i < arr.length(); i++) {
+ for (int i = 0; i < arr.length(); i++) {
lst.push(arr.getString(i));
}
return lst;
}
public DUUIPipelineComponent withDockerAuth(String name, String password) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(name==null && password == null) {
+ if (name == null && password == null) {
_options.remove(dockerPasswordOptionName);
_options.remove(dockerUsernameOptionName);
return this;
}
- _options.put(dockerPasswordOptionName,password);
- _options.put(dockerUsernameOptionName,name);
+ _options.put(dockerPasswordOptionName, password);
+ _options.put(dockerUsernameOptionName, name);
return this;
}
@@ -304,7 +339,7 @@ public String getDockerAuthUsername() {
public String getDockerAuthUsername(String defaultValue) {
String value = _options.get(dockerUsernameOptionName);
- if(value == null) return defaultValue;
+ if (value == null) return defaultValue;
return value;
}
@@ -314,20 +349,20 @@ public String getDockerAuthPassword() {
public String getDockerAuthPassword(String defaultValue) {
String value = _options.get(dockerPasswordOptionName);
- if(value == null) return defaultValue;
+ if (value == null) return defaultValue;
return value;
}
public DUUIPipelineComponent withDockerRunAfterExit(Boolean b) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(b==null) {
+ if (b == null) {
_options.remove(dockerNoShutdown);
return this;
}
- _options.put(dockerNoShutdown,String.valueOf(b));
+ _options.put(dockerNoShutdown, String.valueOf(b));
return this;
}
@@ -337,20 +372,20 @@ public Boolean getDockerRunAfterExit() {
public Boolean getDockerRunAfterExit(Boolean defaultValue) {
String result = _options.get(dockerNoShutdown);
- if(result == null) return defaultValue;
+ if (result == null) return defaultValue;
return Boolean.parseBoolean(result);
}
public DUUIPipelineComponent withDockerGPU(Boolean b) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(b==null) {
+ if (b == null) {
_options.remove(dockerWithGPU);
return this;
}
- _options.put(dockerWithGPU,String.valueOf(b));
+ _options.put(dockerWithGPU, String.valueOf(b));
return this;
}
@@ -360,31 +395,31 @@ public Boolean getDockerGPU() {
public Boolean getDockerGPU(Boolean defaultValue) {
String result = _options.get(dockerWithGPU);
- if(result == null) return defaultValue;
+ if (result == null) return defaultValue;
return Boolean.parseBoolean(result);
}
public DUUIPipelineComponent withDockerImageName(String imageName) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(imageName==null) {
+ if (imageName == null) {
_options.remove(dockerImageName);
return this;
}
- _options.put(dockerImageName,imageName);
+ _options.put(dockerImageName, imageName);
return this;
}
public DUUIPipelineComponent __internalPinDockerImage(String imageName, String pinName) {
- if(pinName==null) {
+ if (pinName == null) {
System.err.println("Could not add the digest since this image has not been pushed and pulled from a registry V2");
- _options.put(dockerImageName,imageName);
+ _options.put(dockerImageName, imageName);
return this;
}
- _options.put(dockerImageName,pinName);
+ _options.put(dockerImageName, pinName);
return this;
}
@@ -394,22 +429,22 @@ public String getDockerImageName() {
public String getDockerImageName(String defaultValue) {
String result = _options.get(dockerImageName);
- if(result == null) {
+ if (result == null) {
return defaultValue;
}
return result;
}
public DUUIPipelineComponent withDockerImageFetching(Boolean b) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(b==null) {
+ if (b == null) {
_options.remove(dockerImageFetching);
return this;
}
- _options.put(dockerImageFetching,String.valueOf(b));
+ _options.put(dockerImageFetching, String.valueOf(b));
return this;
}
@@ -419,32 +454,32 @@ public Boolean getDockerImageFetching() {
public Boolean getDockerImageFetching(Boolean defaultValue) {
String result = _options.get(dockerImageFetching);
- if(result == null) return defaultValue;
+ if (result == null) return defaultValue;
return Boolean.parseBoolean(result);
}
public DUUIPipelineComponent withWriteToView(String viewName) {
- return withWriteToView(viewName,false);
+ return withWriteToView(viewName, false);
}
public DUUIPipelineComponent withWriteToView(String viewName, boolean createViewFromInitialView) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
- if(viewName==null) {
+ if (viewName == null) {
_options.remove(writeToViewName);
_options.remove(initialViewFromInitialViewName);
return this;
}
- _options.put(writeToViewName,viewName);
- _options.put(initialViewFromInitialViewName,String.valueOf(createViewFromInitialView));
+ _options.put(writeToViewName, viewName);
+ _options.put(initialViewFromInitialViewName, String.valueOf(createViewFromInitialView));
return this;
}
public Boolean getCreateViewFromInitialView() {
String value = _options.get(initialViewFromInitialViewName);
- if(value == null) return null;
+ if (value == null) return null;
return Boolean.valueOf(value);
}
@@ -454,34 +489,37 @@ public String getViewName() {
public String getViewName(String defaultValue) {
String value = _options.get(writeToViewName);
- if(value==null) return defaultValue;
+ if (value == null) return defaultValue;
return value;
}
public String toJson() {
JSONObject js = new JSONObject();
- js.put("options",_options);
- js.put("parameters",_parameters);
+ js.put("options", _options);
+ js.put("parameters", _parameters);
return js.toString();
}
/**
* @author Dawit Terefe (edited)
- *
+ *
* Option to choose websocket as protocol.
* Default is false.
* Option to choose number of elements for
* partition size.
- *
*/
- public boolean isWebsocket() { return _websocket; }
+ public boolean isWebsocket() {
+ return _websocket;
+ }
public DUUIPipelineComponent withWebsocket(boolean b) {
_websocket = b;
return withParameter(websocketOptionName, String.valueOf(b));
}
- public int getWebsocketElements () { return _ws_elements; }
+ public int getWebsocketElements() {
+ return _ws_elements;
+ }
public DUUIPipelineComponent withWebsocket(boolean b, int elements) {
_websocket = b;
@@ -491,11 +529,11 @@ public DUUIPipelineComponent withWebsocket(boolean b, int elements) {
}
public DUUIPipelineComponent withParameter(String key, String value) {
- _parameters.put(key,value);
+ _parameters.put(key, value);
return this;
}
- public DUUIPipelineComponent withView(String viewName){
+ public DUUIPipelineComponent withView(String viewName) {
withSourceView(viewName);
withTargetView(viewName);
return this;
@@ -520,18 +558,18 @@ public DUUIPipelineComponent withTimeout(long lLong) {
public static DUUIPipelineComponent fromJson(String json) throws URISyntaxException, IOException {
JSONObject jobj = new JSONObject(json);
- HashMap optionsMap = new HashMap<>();
+ HashMap optionsMap = new HashMap<>();
JSONObject options = jobj.getJSONObject("options");
for (Iterator it = options.keys(); it.hasNext(); ) {
String key = it.next();
- optionsMap.put(key,options.getString(key));
+ optionsMap.put(key, options.getString(key));
}
- HashMap parametersMap = new HashMap<>();
+ HashMap parametersMap = new HashMap<>();
JSONObject parameters = jobj.getJSONObject("parameters");
for (Iterator it = parameters.keys(); it.hasNext(); ) {
String key = it.next();
- parametersMap.put(key,parameters.getString(key));
+ parametersMap.put(key, parameters.getString(key));
}
DUUIPipelineComponent comp = new DUUIPipelineComponent();
@@ -541,58 +579,54 @@ public static DUUIPipelineComponent fromJson(String json) throws URISyntaxExcept
}
public static DUUIPipelineComponent fromEncodedJson(String encodedJson) throws URISyntaxException, IOException, CompressorException {
- return fromEncodedJson(encodedJson,compressionMethod);
+ return fromEncodedJson(encodedJson, compressionMethod);
}
public static DUUIPipelineComponent fromEncodedJson(String encodedJson, String compressionUsed) throws URISyntaxException, IOException, CompressorException {
byte[] decoded = Base64.getDecoder().decode(encodedJson.getBytes(StandardCharsets.UTF_8));
- CompressorInputStream stream = new CompressorStreamFactory().createCompressorInputStream(compressionUsed,new ByteArrayInputStream(decoded));
- String json = new String(stream.readAllBytes(),StandardCharsets.UTF_8);
+ CompressorInputStream stream = new CompressorStreamFactory().createCompressorInputStream(compressionUsed, new ByteArrayInputStream(decoded));
+ String json = new String(stream.readAllBytes(), StandardCharsets.UTF_8);
return fromJson(json);
}
public DUUIPipelineComponent join(DUUIPipelineComponent comp, DUUIPipelineComponentJoinStrategy strategy) {
- if(_finalizedEncoded!=null) {
+ if (_finalizedEncoded != null) {
throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
}
-
- if(strategy == DUUIPipelineComponentJoinStrategy.LEFT_WINS) {
- for(String i : comp._options.keySet()) {
+ if (strategy == DUUIPipelineComponentJoinStrategy.LEFT_WINS) {
+ for (String i : comp._options.keySet()) {
//Discard keys we have too write others into us
- if(!_options.containsKey(i)) {
- _options.put(i,comp._options.get(i));
+ if (!_options.containsKey(i)) {
+ _options.put(i, comp._options.get(i));
}
}
- }
- else if(strategy == DUUIPipelineComponentJoinStrategy.RIGHT_WINS) {
+ } else if (strategy == DUUIPipelineComponentJoinStrategy.RIGHT_WINS) {
//Write everything since we loose on conflict and need missing options anyways
- for(String i : comp._options.keySet()) {
- _options.put(i,comp._options.get(i));
+ for (String i : comp._options.keySet()) {
+ _options.put(i, comp._options.get(i));
}
- }
- else {
- for(String i : comp._options.keySet()) {
- if(_options.containsKey(i)) {
- if(!_options.get(i).equals(comp._options.get(i))) {
+ } else {
+ for (String i : comp._options.keySet()) {
+ if (_options.containsKey(i)) {
+ if (!_options.get(i).equals(comp._options.get(i))) {
throw new RuntimeException(String.format("Key conflict in join in key %s", i));
}
- }
- else {
- _options.put(i,comp._options.get(i));
+ } else {
+ _options.put(i, comp._options.get(i));
}
}
}
return this;
}
- public final Map getParameters() {
+ public final Map getParameters() {
return _parameters;
}
public String getSourceView() {
String result = _options.get(sourceView);
- if(result == null) {
+ if (result == null) {
return "_InitialView";
}
return result;
@@ -600,7 +634,7 @@ public String getSourceView() {
public String getTargetView() {
String result = _options.get(targetView);
- if(result == null) {
+ if (result == null) {
return "_InitialView";
}
return result;
@@ -632,25 +666,21 @@ public DUUIUIMADriver.Component asUIMADriverComponent() throws URISyntaxExceptio
}
public String attemptAutomaticDescription() throws InvalidXMLException, URISyntaxException, IOException, SAXException {
- if(getDriver().equals(DUUIUIMADriver.class.getCanonicalName())) {
- return String.format("UIMA annotator: %s, scale: %d",asUIMADriverComponent().getAnnotatorName(),getScale(1));
- }
- else if(getDriver().equals(DUUIDockerDriver.class.getCanonicalName())) {
- return String.format("Docker annotator: %s, scale: %d",getDockerImageName(),getScale(1));
- }
- else if(getDriver().equals(DUUISwarmDriver.class.getCanonicalName())) {
- return String.format("Swarm annotator: %s, scale: %d",getDockerImageName(),getScale(1));
- }
- else if(getDriver().equals(DUUIRemoteDriver.class.getCanonicalName())) {
+ if (getDriver().equals(DUUIUIMADriver.class.getCanonicalName())) {
+ return String.format("UIMA annotator: %s, scale: %d", asUIMADriverComponent().getAnnotatorName(), getScale(1));
+ } else if (getDriver().equals(DUUIDockerDriver.class.getCanonicalName())) {
+ return String.format("Docker annotator: %s, scale: %d", getDockerImageName(), getScale(1));
+ } else if (getDriver().equals(DUUISwarmDriver.class.getCanonicalName())) {
+ return String.format("Swarm annotator: %s, scale: %d", getDockerImageName(), getScale(1));
+ } else if (getDriver().equals(DUUIRemoteDriver.class.getCanonicalName())) {
String urls = "[";
- for(String x : getUrl()) {
- urls+=x;
- urls+=",";
+ for (String x : getUrl()) {
+ urls += x;
+ urls += ",";
}
- urls = urls.substring(0,urls.length()-1)+"]";
- return String.format("Remote annotator: %s, scale: %d",urls,getScale(1));
- }
- else {
+ urls = urls.substring(0, urls.length() - 1) + "]";
+ return String.format("Remote annotator: %s, scale: %d", urls, getScale(1));
+ } else {
return "Unkown annotator driver!";
}
}
@@ -667,4 +697,274 @@ public DUUISegmentationStrategy getSegmentationStrategy() {
public long getTimeout() {
return Long.valueOf(_parameters.getOrDefault(timeout, "60"));
}
+
+ public DUUIPipelineComponent withSlurmJobName(String jobName) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (jobName == null) {
+ _options.remove(slurmJobName);
+ return this;
+ }
+ _options.put(slurmJobName, jobName);
+ _options.put(componentName, jobName);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmHostPort(String port) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (port == null) {
+ _options.remove(slurmHostPort);
+ return this;
+ }
+ _options.put(slurmHostPort, port);
+ return this;
+ }
+
+
+ public DUUIPipelineComponent withSlurmRuntime(String time) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (time == null) {
+ _options.remove(slurmRuntime);
+ return this;
+ }
+ _options.put(slurmRuntime, time);
+ return this;
+ }
+
+
+ public DUUIPipelineComponent withSlurmMemory(String numMB) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (numMB == null) {
+ _options.remove(slurmMemory);
+ return this;
+ }
+ _options.put(slurmMemory, numMB);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmOutPutLocation(String loc) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (loc == null) {
+ _options.remove(slurmOutPutLocation);
+ return this;
+ }
+ _options.put(slurmOutPutLocation, loc);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmErrorLocation(String loc) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (loc == null) {
+ _options.remove(slurmErrorLocation);
+ return this;
+ }
+ _options.put(slurmErrorLocation, loc);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmSaveIn(String saveTo) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (saveTo == null) {
+ _options.remove(slurmSIFLocation);
+ return this;
+ }
+ _options.put(slurmSIFLocation, saveTo);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmGPU(String num) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (num == null) {
+ _options.remove(slurmGPU);
+ return this;
+ }
+ _options.put(slurmGPU, num);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmCPUs(String num) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (num == null) {
+ _options.remove(slurmCpus);
+ return this;
+ }
+ _options.put(slurmCpus, num);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmSIFName(String sifName) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (sifName == null) {
+ _options.remove(slurmSIFImageName);
+ return this;
+ }
+ _options.put(slurmSIFImageName, sifName);
+ return this;
+
+ }
+
+
+ public DUUIPipelineComponent withSlurmUvicorn(String loc) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (loc == null) {
+ _options.remove(slurmUvicorn);
+ return this;
+ }
+ _options.put(slurmUvicorn, loc);
+ return this;
+ }
+
+
+ public DUUIPipelineComponent withSlurmScript(String script) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (script == null) {
+ _options.remove(slurmScript);
+ return this;
+ }
+ _options.put(slurmScript, script);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmPartition(String pname) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (pname == null) {
+ _options.remove(slurmPartition);
+ return this;
+ }
+ _options.put(slurmPartition, pname);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmNodelist(String nodelist) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (nodelist == null) {
+ _options.remove(slurmNodelist);
+ return this;
+ }
+ _options.put(slurmNodelist, nodelist);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmWorkDir(String wd) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (wd == null) {
+ _options.remove(slurmWorkDir);
+ return this;
+ }
+ _options.put(slurmWorkDir, wd);
+ return this;
+ }
+
+ public DUUIPipelineComponent withSlurmInnerPort(String port) {
+ if (_finalizedEncoded != null) {
+ throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!");
+ }
+ if (port == null) {
+ _options.remove(slurmInnerPort);
+ return this;
+ }
+ _options.put(slurmInnerPort, port);
+ return this;
+ }
+
+
+ public String getSlurmSIFImageName() {
+ return _options.get(slurmSIFImageName);
+ }
+
+ public String getSlurmGPU() {
+ return _options.get(slurmGPU);
+ }
+
+ public String getSlurmSIFLocation() {
+ return _options.get(slurmSIFLocation);
+ }
+
+ public String getSlurmOutPutLocation() {
+ return _options.get(slurmOutPutLocation);
+ }
+
+ public String getSlurmMem() {
+ return _options.get(slurmMemory);
+ }
+
+ public String getSlurmRuntime() {
+ return _options.get(slurmRuntime);
+ }
+
+ public String getSlurmHostPort() {
+ return _options.get(slurmHostPort);
+ }
+
+ public String getSlurmJobName() {
+ return _options.get(slurmJobName);
+ }
+
+ public String getSlurmErrorLocation() {
+ return _options.get(slurmErrorLocation);
+ }
+
+ public Boolean getSlurmRunAfterExit(Boolean defaultValue) {
+ String result = _options.get(slurmNoShutdown);
+ if (result == null) return defaultValue;
+ return Boolean.parseBoolean(result);
+ }
+
+ public String getSlurmUvicorn() {
+ return _options.get(slurmUvicorn);
+ }
+
+ public String getSlurmScript() {
+ return _options.get(slurmScript);
+ }
+
+ public String getSlurmPartition() {
+ return _options.get(slurmPartition);
+ }
+
+ public String getSlurmNodelist() {
+ return _options.get(slurmNodelist);
+ }
+
+ public String getSlurmWorkDir() {
+ return _options.get(slurmWorkDir);
+ }
+
+ public String getSlurmInnerPort() {
+ return _options.get(slurmInnerPort);
+ }
+
+ public String getSlurmCpus() {
+ return _options.get(slurmCpus);
+ }
+
}
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java
index 8d48e53a..a652f15a 100644
--- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java
@@ -98,11 +98,11 @@ public interface IDUUIDriverInterface {
* @param uuid
* @return
*/
- public boolean destroy(String uuid);
+ public boolean destroy(String uuid) throws IOException, InterruptedException;
/**
* Shutting down the driver
*/
- public void shutdown();
+ public void shutdown() throws IOException, InterruptedException;
}
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java
new file mode 100755
index 00000000..223fe552
--- /dev/null
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java
@@ -0,0 +1,642 @@
+package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm;
+
+import io.vertx.core.cli.annotations.Hidden;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.uima.UIMAException;
+import org.apache.uima.cas.CASException;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.metadata.TypeSystemDescription;
+import org.apache.uima.util.TypeSystemUtil;
+import org.javatuples.Triplet;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIDockerInterface;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIFallbackCommunicationLayer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.IDUUICommunicationLayer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.DUUIWebsocketAlt;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.IDUUIConnectionHandler;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.*;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.exception.PipelineComponentException;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaCommunicationLayer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.pipeline_storage.DUUIPipelineDocumentPerformance;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.segmentation.DUUISegmentationStrategy;
+import org.xml.sax.SAXException;
+
+import javax.sound.sampled.Port;
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.security.InvalidParameterException;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Logger;
+
+import static java.lang.String.format;
+
+
+public class DUUISlurmDriver implements IDUUIDriverInterface {
+ private SlurmRest _rest;
+ private DUUIDockerInterface _docker_interface;
+ private DUUILuaContext _luaContext;
+ private DUUISlurmInterface _interface;
+
+ private int _http_timeout_s = 10;
+ private HashMap _active_components;
+ private HttpClient _client;
+ private IDUUIConnectionHandler _wsclient;
+ private final static Logger LOGGER = Logger.getLogger(DUUIComposer.class.getName());
+
+
+ public DUUISlurmDriver(SlurmRest rest) throws IOException, UIMAException, SAXException {
+ _rest = rest;
+ _interface = new DUUISlurmInterface(rest);
+ _client = HttpClient.newHttpClient();
+
+ JCas _basic = JCasFactory.createJCas();
+ _basic.setDocumentLanguage("en");
+ _basic.setDocumentText("Hello World!");
+ //_http_timeout = 10000;
+ TypeSystemDescription desc = TypeSystemUtil.typeSystem2TypeSystemDescription(_basic.getTypeSystem());
+ StringWriter wr = new StringWriter();
+ // trigger for type checking
+ desc.toXML(wr);
+ _active_components = new HashMap();
+ _luaContext = null;
+ }
+
+ /**
+ * Creation of the communication layer based on the Driver
+ *
+ * @param url localhost
+ * @param jc jcas
+ * @param http_timeout_s timeout with fastapi
+ * @param client communication wtih fastapi
+ * @param context lua
+ * @param skipVerification true
+ * @return
+ * @throws Exception
+ */
+ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, int http_timeout_s, HttpClient client, DUUILuaContext context, boolean skipVerification) throws Exception {
+ long start = System.currentTimeMillis();
+ // default http
+ IDUUICommunicationLayer layer = new DUUIFallbackCommunicationLayer(); // Hier wird layer zum ersten mal erstellt.
+ boolean fatal_error = false;
+
+ // error counter
+ int iError = 0;
+ while (true) {
+ HttpRequest request = null;
+ try {// req
+ // get lua script
+ request = HttpRequest.newBuilder()
+ .uri(URI.create(url + DUUIComposer.V1_COMPONENT_ENDPOINT_COMMUNICATION_LAYER))
+ .version(HttpClient.Version.HTTP_1_1)
+// .timeout(Duration.ofSeconds(10))
+ .timeout(Duration.ofSeconds(http_timeout_s))
+ .GET()
+ .build();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ try {
+ HttpResponse resp = null;
+
+ boolean connectionError = true;
+ int iCount = 0;
+ //max.10
+ while (connectionError && iCount < 10) {
+
+ try {
+
+ // req lua
+ resp = client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join();
+
+ connectionError = false;
+ } catch (Exception e) {
+ System.out.println(e.getMessage() + "\t" + url);
+ if (e instanceof java.net.ConnectException) {
+ Thread.sleep(http_timeout_s);
+ iCount++;
+ } else if (e instanceof CompletionException) {
+ Thread.sleep(http_timeout_s);
+ iCount++;
+ }
+ }
+ }
+ // if success?
+ if (resp.statusCode() == 200) {
+ // parse script as string
+ String body2 = new String(resp.body(), Charset.defaultCharset());
+ try {
+ //printfunc.operation("Component lua communication layer, loading...");
+ // new lua commu layer
+ IDUUICommunicationLayer lua_com = new DUUILuaCommunicationLayer(body2, "requester", context);
+ // replace
+ layer = lua_com;
+
+ //printfunc.operation("Component lua communication layer, loaded.");
+ break;
+ } catch (Exception e) {
+ fatal_error = true;
+ e.printStackTrace();
+ throw new Exception("Component provided a lua script which is not runnable.");
+ }
+ // fallback
+ } else if (resp.statusCode() == 404) {
+ // printfunc.operation("Component provided no own communication layer implementation using fallback.");
+ break;
+ }
+ long finish = System.currentTimeMillis();
+ long timeElapsed = finish - start;
+ if (timeElapsed > http_timeout_s) {
+ throw new TimeoutException(format("The Container did not provide one succesful answer in %d milliseconds", http_timeout_s));
+ }
+
+ } catch (Exception e) {
+
+ if (fatal_error) {
+ throw e;
+ } else {
+ Thread.sleep(2000l);
+ iError++;
+ }
+
+ if (iError > 10) {
+ throw e;
+ }
+ }
+ }
+ if (skipVerification) {
+ return layer;
+ }
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+ try {
+
+ layer.serialize(jc, stream, null, "_InitialView");
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new Exception(format("The serialization step of the communication layer fails for implementing class %s", layer.getClass().getCanonicalName()));
+ }
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create(url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS))
+ .version(HttpClient.Version.HTTP_1_1)
+ .POST(HttpRequest.BodyPublishers.ofByteArray(stream.toByteArray()))
+ .build();
+ HttpResponse resp = client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join();
+
+ if (resp.statusCode() == 200) {
+ ByteArrayInputStream inputStream = new ByteArrayInputStream(resp.body());
+ try {
+ layer.deserialize(jc, inputStream, "_InitialView");
+ } catch (Exception e) {
+ System.err.printf("Caught exception printing response %s\n", new String(resp.body(), StandardCharsets.UTF_8));
+ throw e;
+ }
+ return layer;
+ } else {
+ throw new Exception(format("The container returned response with code != 200\nResponse %s", resp.body().toString()));
+ }
+ }
+
+
+ public void setLuaContext(DUUILuaContext luaContext) {
+ _luaContext = luaContext;
+ }
+
+
+ public boolean canAccept(DUUIPipelineComponent comp) {
+
+ return comp.getSlurmScript()!= null || !comp.getSlurmRuntime().isEmpty();
+
+
+ }
+
+
+ /**
+ * The original code has changed a lot, so please check the interlinear comments.
+ * @param component
+ * @param jc
+ * @param skipVerification
+ * @param shutdown
+ * @return
+ * @throws InterruptedException
+ * @throws URISyntaxException
+ * @throws IOException
+ */
+ public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skipVerification, AtomicBoolean shutdown) throws InterruptedException, URISyntaxException, IOException {
+ String uuid = UUID.randomUUID().toString();
+ while (_active_components.containsKey(uuid.toString())) {
+ uuid = UUID.randomUUID().toString();
+ }
+
+
+ DUUISlurmDriver.InstantiatedComponent comp = new DUUISlurmDriver.InstantiatedComponent(component);
+
+ System.out.printf("[SlurmDriver] Assigned new pipeline component unique id %s\n", uuid);
+
+ _active_components.put(uuid, comp);
+
+ for (int i = 0; i < comp.getScale(); i++) {
+ if (shutdown.get()) {
+ return null;
+ }
+ // If only one component is started, the user can specify a port on their own using withSlurmHostPort()
+ // Otherwise it will be assigned by portmanager
+ int hostPort = -1;
+ String slurmHostPort = component.getSlurmHostPort();
+ if (slurmHostPort == null || slurmHostPort.isEmpty()) {
+ System.out.println("[SlurmDriver] Assigning a free port follows the FIFO-ALGORITHM");
+ hostPort = PortManager.acquire();
+ } else {
+ System.out.println("[SlurmDriver] Assigning a port of your choice.");
+ hostPort = Integer.parseInt(slurmHostPort);
+ }
+ //Users can generate scripts from the methods in SlurmUtil by entering a series of parameters,
+ // or they can just enter a script in json format.
+ System.out.println("[SlurmDriver] prepare to reserve port: " + hostPort);
+ String job_id;
+ if (component.getSlurmScript() != null) {
+ String jobID = _interface.run_json(component, hostPort);
+ job_id = jobID;
+ System.out.println("[SlurmDriver] Submit successfully, Job ID: " + jobID);
+
+
+ } else {
+ //
+ String jobID = _interface.run(component, hostPort);
+
+ job_id = jobID;
+ System.out.println("[SlurmDriver] submit " + component.getSlurmJobName() + " successfully, Job ID: " + jobID);
+ }
+ // Since submitting tasks in slurm is asynchronous,
+ // the java code must be blocked to continue running, which prevents http error
+ boolean ok = PortManager.waitUntilHttpReachable(
+ "http://localhost:" + hostPort + "/v1/communication_layer",
+ 200,
+ null,
+ Duration.ofMinutes(1),
+ Duration.ofSeconds(2)
+ );
+
+ if (!ok) throw new IllegalStateException("timeout waiting for http ");
+
+ // DUUISlurmInterface maintains a thread-safe hashmap, see DUUISlurmInterface.class for details.
+ String portFromMap = _interface.extractPort(job_id);
+ if (!portFromMap.equals(Integer.toString(hostPort))) {
+ throw new RuntimeException("port mismatch");
+ }
+
+
+ try {
+ final int iCopy = i;
+ final String uuidCopy = uuid;
+ IDUUICommunicationLayer layer = responsiveAfterTime("http://127.0.0.1" + ":" + hostPort, jc, _http_timeout_s, _client, _luaContext, skipVerification);
+ if (component.getScale()==null)
+ {
+ System.out.println("[SlurmDriver] " + component.getSlurmSIFImageName() + "[" + Integer.toString(iCopy + 1) + "/" + "1" + "]" + "is online and seems to understand DUUI V1 format!");
+
+ }
+
+ System.out.println("[SlurmDriver] " + component.getSlurmSIFImageName() + "[" + Integer.toString(iCopy + 1) + "/" + component.getScale() + "]" + "is online and seems to understand DUUI V1 format!");
+ if (comp.isWebsocket()) {
+ String url = "ws://127.0.0.1:" + hostPort;
+ _wsclient = new DUUIWebsocketAlt(
+ url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS_WEBSOCKET, comp.getWebsocketElements());
+ } else {
+ _wsclient = null;
+ }
+
+ comp.addInstance(new DUUISlurmDriver.ComponentInstance(job_id, hostPort, layer, _wsclient));
+ } catch (Exception e) {
+
+ }
+
+ }
+ return shutdown.get() ? null : uuid;
+ }
+
+
+ /**
+ * Show the maximum parallelism
+ *
+ * @param uuid
+ */
+ @Override
+ public void printConcurrencyGraph(String uuid) {
+ DUUISlurmDriver.InstantiatedComponent component = _active_components.get(uuid);
+ if (component == null) {
+ throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver");
+ }
+ System.out.printf("[DockerLocalDriver][%s]: Maximum concurrency %d\n", uuid, component.getInstances().size());
+ }
+
+
+ /**
+ * Return the TypeSystem used by the given Component
+ *
+ * @param uuid
+ * @return
+ * @throws InterruptedException
+ * @throws IOException
+ * @throws SAXException
+ * @throws CompressorException
+ * @throws ResourceInitializationException
+ */
+ public TypeSystemDescription get_typesystem(String uuid) throws InterruptedException, IOException, SAXException, CompressorException, ResourceInitializationException {
+ DUUISlurmDriver.InstantiatedComponent comp = _active_components.get(uuid);
+ if (comp == null) {
+ throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver");
+ }
+ return IDUUIInstantiatedPipelineComponent.getTypesystem(uuid, comp);
+ }
+
+
+ /**
+ * init reader component
+ *
+ * @param uuid
+ * @param filePath
+ * @return
+ */
+ @Override
+ public int initReaderComponent(String uuid, Path filePath) {
+ InstantiatedComponent comp = _active_components.get(uuid);
+ if (comp == null) {
+ throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver");
+ }
+ return IDUUIInstantiatedPipelineReaderComponent.initComponent(comp, filePath);
+ }
+//-------------------------------------------------------------------------------------------------------------
+
+
+ public static class ComponentInstance implements IDUUIUrlAccessible {
+ private String _job_ID;
+ private int _hostPort;
+ private IDUUIConnectionHandler _handler;
+ private IDUUICommunicationLayer _communicationLayer;
+
+ public ComponentInstance(String id, int hostPort, IDUUICommunicationLayer communicationLayer) {
+ _job_ID = id;
+ _hostPort = hostPort;
+ _communicationLayer = communicationLayer;
+ }
+
+ public ComponentInstance(String id, int hostPort, IDUUICommunicationLayer communicationLayer, IDUUIConnectionHandler handler) {
+ _job_ID = id;
+ _hostPort = hostPort;
+
+ _communicationLayer = communicationLayer;
+ _handler = handler;
+ }
+
+ public IDUUICommunicationLayer getCommunicationLayer() {
+ return _communicationLayer;
+ }
+
+
+ public String get_job_ID() {
+ return _job_ID;
+ }
+
+ public int get_hostPort() {
+ return _hostPort;
+ }
+
+
+ public IDUUIConnectionHandler get_handler() {
+ return _handler;
+ }
+
+ public IDUUICommunicationLayer get_communicationLayer() {
+ return _communicationLayer;
+ }
+
+ public String generateURL() {
+ return format("http://127.0.0.1" + ":%d", _hostPort);
+ }
+
+ public String generateURL(String localhost) {
+ return format("http://localhost" + ":%d", _hostPort);
+ }
+
+ public IDUUIConnectionHandler getHandler() {
+ return _handler;
+ }
+ }
+
+ /**
+ * Execute a component in the driver
+ *
+ * @param uuid
+ * @param aCas
+ * @param perf
+ * @param composer
+ * @throws CASException
+ */
+ public void run(String uuid, JCas aCas, DUUIPipelineDocumentPerformance perf, DUUIComposer composer) throws CASException, PipelineComponentException {
+ long mutexStart = System.nanoTime();
+ DUUISlurmDriver.InstantiatedComponent comp = _active_components.get(uuid);
+ if (comp == null) {
+ throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver");
+ }
+ if (comp.isWebsocket()) {
+ IDUUIInstantiatedPipelineComponent.process_handler(aCas, comp, perf);
+ } else {
+ IDUUIInstantiatedPipelineComponent.process(aCas, comp, perf);
+ }
+ }
+
+
+ static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent {
+ private String _image_name;
+ private ConcurrentLinkedQueue _instances;
+ private boolean _gpu;
+ private boolean _keep_runnging_after_exit;
+ private int _scale;
+ private boolean _withImageFetching;
+ private boolean _websocket;
+ private int _ws_elements;
+ private String _uniqueComponentKey;
+ private Map _parameters;
+ private String _sourceView;
+ private String _targetView;
+ private DUUIPipelineComponent _component;
+
+ public Triplet getComponent() {
+ long mutexStart = System.nanoTime();
+ DUUISlurmDriver.ComponentInstance inst = _instances.poll();
+ while (inst == null) {
+ inst = _instances.poll();
+ }
+ long mutexEnd = System.nanoTime();
+ return Triplet.with(inst, mutexStart, mutexEnd);
+ }
+
+ public void addComponent(IDUUIUrlAccessible access) {
+ _instances.add((DUUISlurmDriver.ComponentInstance) access);
+ }
+
+ InstantiatedComponent(DUUIPipelineComponent comp) {
+ _component = comp;
+ _parameters = comp.getParameters();
+ _targetView = comp.getTargetView();
+ _sourceView = comp.getSourceView();
+
+ _withImageFetching = comp.getDockerImageFetching(false);
+
+ _uniqueComponentKey = "";
+
+
+ _instances = new ConcurrentLinkedQueue();
+
+ _scale = comp.getScale(1);
+
+ _gpu = comp.getDockerGPU(false);
+
+ _keep_runnging_after_exit = comp.getSlurmRunAfterExit(false);
+
+ _websocket = comp.isWebsocket();
+ _ws_elements = comp.getWebsocketElements();
+ }
+
+ public DUUIPipelineComponent getPipelineComponent() {
+ return _component;
+ }
+
+ public String getUniqueComponentKey() {
+ return _uniqueComponentKey;
+ }
+
+ public boolean getImageFetching() {
+ return _withImageFetching;
+ }
+
+ public String getImageName() {
+ return _image_name;
+ }
+
+ public int getScale() {
+ return _scale;
+ }
+
+ public boolean getRunningAfterExit() {
+ return _keep_runnging_after_exit;
+ }
+
+ public void addInstance(DUUISlurmDriver.ComponentInstance inst) {
+ _instances.add(inst);
+ }
+
+ public boolean usesGPU() {
+ return (Integer.parseInt(_component.getSlurmGPU()) > 0) ? true : false;
+ }
+
+ public ConcurrentLinkedQueue getInstances() {
+ return _instances;
+ }
+
+ public Map getParameters() {
+ return _parameters;
+ }
+
+ public String getSourceView() {
+ return _sourceView;
+ }
+
+ public String getTargetView() {
+ return _targetView;
+ }
+
+ public boolean isWebsocket() {
+ return _websocket;
+ }
+
+ public int getWebsocketElements() {
+ return _ws_elements;
+ }
+ }
+
+
+ public static class Component {
+ private DUUIPipelineComponent _component;
+
+
+ public Component(DUUIPipelineComponent pComponent) throws URISyntaxException, IOException {
+ _component = pComponent;
+ _component.withDriver(DUUISlurmDriver.class);
+ }
+
+ public DUUIPipelineComponent build() {
+ return _component;
+ }
+
+ }
+
+ @Override
+ public void shutdown() throws IOException, InterruptedException{
+ String token = _rest.generateRootToken("slurmctld");
+ System.out.println("shutdown hook clean all jobs");
+ HashMap jobIDPortMap = _interface.get_jobID_PortMap();
+ jobIDPortMap.keySet().forEach(jobID -> {
+ try {
+ _rest.cancelJob(token, jobID);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+
+ /**
+ * Because it is a self-maintained port table,
+ * it allows the port to be released to another component for continued use after the end of each type of component.
+ * See PortManager.class for details
+ * @param uuid
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Override
+ public boolean destroy(String uuid) throws IOException, InterruptedException {
+
+ String token = _rest.generateRootToken("slurmctld");
+ DUUISlurmDriver.InstantiatedComponent comp = _active_components.remove(uuid);
+ boolean flag = true;
+ if (comp == null) {
+ throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver");
+ }
+ if (!comp.getRunningAfterExit()) {
+ int counter = 1;
+ for (DUUISlurmDriver.ComponentInstance inst : comp.getInstances()) {
+ String jobId = inst.get_job_ID();
+ boolean status = _rest.cancelJob(token, jobId);
+ PortManager.release(inst.get_hostPort());
+ _interface.get_jobID_PortMap().remove(jobId);
+ counter += 1;
+ flag = flag && status;
+
+ }
+ }
+ return flag;
+ }
+}
+
+
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java
new file mode 100755
index 00000000..bb6c4813
--- /dev/null
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java
@@ -0,0 +1,101 @@
+package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.core.DockerClientBuilder;
+import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
+import com.github.dockerjava.transport.DockerHttpClient;
+import com.influxdb.client.JSON;
+import org.json.JSONObject;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest;
+
+import java.io.IOException;
+import java.net.URI;
+import java.time.Duration;
+import java.util.HashMap;
+
+
+public class DUUISlurmInterface {
+
+ private SlurmRest slurmRest;
+
+ private HashMap _jobID_PortMap = new HashMap<>();
+
+ public DUUISlurmInterface(SlurmRest rest) throws IOException {
+ slurmRest = rest;
+ }
+
+ /**
+ * Generate a script based on the parameters given by the user, submit and execute it
+ * @param comp
+ * @param hostPort
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String run(DUUIPipelineComponent comp, int hostPort) throws IOException, InterruptedException {
+ String hport = Integer.toString(hostPort);
+ String gpu = comp.getSlurmGPU();
+ String sifName = comp.getSlurmSIFImageName();
+ String slurmJobName = comp.getSlurmJobName();
+ String slurmRuntime = comp.getSlurmRuntime();
+ String slurmMem = comp.getSlurmMem();
+ String slurmSaveIn = comp.getSlurmSIFLocation();
+ String slurmUvicorn = comp.getSlurmUvicorn();
+ String slurmNodelist = comp.getSlurmNodelist();
+ String slurmPartition = comp.getSlurmPartition();
+ String slurmWorkDir = comp.getSlurmWorkDir();
+ String slurmCpus = comp.getSlurmCpus();
+ JSONObject j = new JSONObject();
+ if (Integer.parseInt(gpu)>0){
+ JSONObject jsonObject = SlurmUtils.generateJobScript_GPU(slurmJobName, slurmRuntime, slurmWorkDir, hport, slurmSaveIn, slurmUvicorn, slurmMem, gpu, sifName, slurmPartition, slurmNodelist, slurmCpus);
+ j = jsonObject;
+ }
+ else {
+ j = SlurmUtils.generateJobScript_non_GPU(slurmJobName,slurmRuntime,slurmWorkDir,hport,slurmSaveIn, slurmUvicorn,slurmMem,sifName,slurmPartition,slurmNodelist, slurmCpus);
+
+ }
+ String token = slurmRest.generateRootToken("slurmctld");
+ String[] submit = slurmRest.submit(token, j);
+ //System.out.println(submit[0]);
+ String id = submit[1];
+ // System.out.println("[SlurmDriver] Job submitted , jobid : " + id);
+ _jobID_PortMap.put(id, Integer.toString(hostPort));
+
+ return id;
+ }
+
+ /**
+ * submit and execute json script
+ * @param comp
+ * @param hostPort
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String run_json(DUUIPipelineComponent comp, int hostPort) throws IOException, InterruptedException {
+ String slurmScript = comp.getSlurmScript();
+ JSONObject slurmJson = new JSONObject(slurmScript);
+ if (slurmRest.checkRESTD()) {
+ String token = slurmRest.generateRootToken("slurmctld");
+ String[] submit = slurmRest.submit(token, slurmJson);
+ //System.out.println(submit[0]);
+ _jobID_PortMap.put(submit[1], Integer.toString(hostPort));
+ return submit[1];
+ }
+ else {
+ throw new RuntimeException("slurm rest check failed");
+ }
+ }
+
+
+ public String extractPort(String jobid) {
+ return _jobID_PortMap.get(jobid);
+ }
+
+ public HashMap get_jobID_PortMap() {
+ return _jobID_PortMap;
+ }
+
+}
+
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java
new file mode 100755
index 00000000..8802d19b
--- /dev/null
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java
@@ -0,0 +1,108 @@
+package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+/**
+ * he port numbers allowed here should be the same as ports:- xx:xx in docker-compose.yml.
+ * Due to the algorithmic multiplexing of ports using fifo,
+ * each component theoretically opens up a maximum of MAX-BASE+1 at the same time.
+ * If there is not enough hardware to allocate, an error is reported.
+ */
+public class PortManager {
+ private static final int BASE = 20000;
+ private static final int MAX = 20100;
+ private static final AtomicInteger next = new AtomicInteger(BASE);
+ private static final ConcurrentLinkedQueue reuse_queue = new ConcurrentLinkedQueue<>();
+
+ public static int acquire() throws IOException {
+ Integer fromPool = reuse_queue.poll();
+ if (fromPool != null) return fromPool;
+
+ int candidate = next.getAndIncrement();
+ if (candidate >= MAX) {
+ throw new IOException("No free port in pool (" + BASE + "‑" + (MAX) + ')');
+ }
+ return candidate;
+ }
+
+ public static void release(int port) {
+ if (port < BASE || port >= MAX) return;
+ reuse_queue.offer(port);
+ }
+
+
+ public static int remaining() {
+ int produced = next.get() - BASE;
+ int total = MAX - BASE;
+ return total - produced + reuse_queue.size();
+ }
+
+ public static boolean isHttpReachable(String url,
+ int expectedStatus,
+ Predicate bodyPredicate,
+ int timeoutMillis) {
+ HttpURLConnection conn = null;
+ try {
+ conn = (HttpURLConnection) new URL(url).openConnection();
+ conn.setConnectTimeout(timeoutMillis);
+ conn.setReadTimeout(timeoutMillis);
+ conn.setRequestMethod("GET");
+
+ int status = conn.getResponseCode();
+ if (status != expectedStatus) {
+ return false;
+ }
+
+ if (bodyPredicate == null) {
+ return true;
+ }
+
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
+ StringBuilder sb = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ sb.append(line);
+ }
+ return bodyPredicate.test(sb.toString());
+ }
+ } catch (IOException e) {
+ return false;
+ } finally {
+ if (conn != null) {
+ conn.disconnect();
+ }
+ }
+ }
+
+
+ public static boolean waitUntilHttpReachable(String url,
+ int expectedStatus,
+ Predicate bodyPredicate,
+ Duration maxWait,
+ Duration interval) throws InterruptedException {
+ Instant deadline = Instant.now().plus(maxWait);
+ while (Instant.now().isBefore(deadline)) {
+ if (isHttpReachable(url, expectedStatus, bodyPredicate,
+ (int) interval.toMillis())) {
+ return true;
+ }
+ Thread.sleep(interval.toMillis());
+ }
+ return false;
+ }
+
+}
+
+
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java
new file mode 100755
index 00000000..9dd3ba4b
--- /dev/null
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java
@@ -0,0 +1,203 @@
+package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm;
+
+import io.swagger.util.Json;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SlurmUtils {
+ public static boolean isDockerImagePresent(String imageName) throws IOException, InterruptedException {
+ ProcessBuilder pb = new ProcessBuilder("docker", "image", "inspect", imageName);
+ Process start = pb.start();
+ return start.waitFor() == 0;
+
+ }
+
+ public static boolean pullSifImagefromRemoteDockerRepo(String dockerImageName, String sifImagePath) {
+ if (sifImagePath == null || sifImagePath.isEmpty()) {
+ throw new IllegalArgumentException("imagePath is null or empty");
+ }
+
+ System.out.println("Correct format: apptainer build {Sif_image_name}.sif docker://{docker_repo/image:tag}");
+
+ List command = List.of(
+ "apptainer", "build",
+ sifImagePath,
+ "docker://" + dockerImageName
+ );
+ ProcessBuilder pb = new ProcessBuilder(command);
+ pb.redirectErrorStream(true);
+ pb.inheritIO();
+ try {
+ Process process = pb.start();
+ int exitCode = process.waitFor();
+ return exitCode == 0;
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Failed to run apptainer build", e);
+ }
+ }
+
+
+ public static boolean pullSifImagefromLocalDockerRepo(String dockerImageName, String sifImagePath) {
+ if (dockerImageName == null || dockerImageName.isEmpty()) {
+ throw new IllegalArgumentException("dockerImageName is null or empty");
+ }
+
+ System.out.println("Correct format: apptainer build {Sif_image_name}.sif docker-daemon://{image:tag}");
+
+ List command = List.of(
+ "apptainer", "build",
+ sifImagePath,
+ "docker-daemon://" + dockerImageName
+ );
+
+ ProcessBuilder pb = new ProcessBuilder(command);
+ pb.redirectErrorStream(true);
+ pb.inheritIO();
+ try {
+ Process process = pb.start();
+ int exitCode = process.waitFor();
+ return exitCode == 0;
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException("Failed to run apptainer build", e);
+ }
+ }
+
+
+
+ public static JSONObject generateJobScript_GPU(
+ String jobname,
+ String runtime,
+ String workdir,
+ String hostPort,
+ String sif_location,
+ String uvicornCmd,
+ String memory,
+ String gpu,
+ String sifimagename,
+ String partitionname,
+ String whichnode,
+ String cpus
+ ) {
+ int minutes = Integer.parseInt(runtime);
+ int mems = Integer.parseInt(memory);
+
+ int outer = Integer.parseInt(hostPort);
+ JSONObject job = new JSONObject()
+ .put("name", jobname)
+ .put("partition", partitionname)
+ .put("time_limit", minutes)
+ .put("current_working_directory", workdir)
+ .put("cpus_per_task", Integer.parseInt(cpus))
+ .put("nodes", "1")
+ .put("required_nodes", List.of(whichnode))
+ .put("memory_per_node", new JSONObject()
+ .put("set", true)
+ .put("number", mems))
+ .put( "tres_per_node", "gres/gpu="+gpu)
+ .put("standard_output", workdir+"/"+sifimagename+"/"+sifimagename+".out")
+ .put("standard_error", workdir+"/"+sifimagename+"/"+sifimagename+".err")
+ .put("environment",
+ new JSONArray().put("PATH=/bin:/usr/bin:/sbin:$PATH"));
+
+ String script = """
+ #!/bin/bash -e
+ find_free() { for p in $(seq 30000 31000); do ss -ltn "( sport = :$p )" | grep -q LISTEN || { echo $p; return; }; done; }
+ INNER=$(find_free)
+ PORT=%d
+ IMG="%s"
+ INTOIMAGE="cd /usr/src/app"
+ UVI="%s"
+
+ apptainer exec "$IMG" \\
+ sh -c "$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER" &
+ PID=$!
+
+ socat TCP-LISTEN:$PORT,bind=0.0.0.0,reuseaddr,fork TCP:127.0.0.1:$INNER &
+ PID_SOCAT=$!
+
+ trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT
+ wait $PID
+ """.formatted(outer, sif_location, uvicornCmd);
+
+ return new JSONObject()
+ .put("script", script)
+ .put("job", job);
+ }
+
+
+
+ public static JSONObject generateJobScript_non_GPU(
+ String jobname,
+ String runtime,
+ String workdir,
+ String hostPort,
+ String sif_location,
+ String uvicornCmd,
+ String memory,
+ // String gpu,
+ String sifimagename,
+ String partitionname,
+ String whichnode,
+ String cpus
+ ) {
+ int minutes = Integer.parseInt(runtime);
+ int mems = Integer.parseInt(memory);
+ int outer = Integer.parseInt(hostPort);
+ JSONObject job = new JSONObject()
+ .put("name", jobname)
+ .put("partition", partitionname)
+ .put("time_limit", minutes)
+ .put("cpus_per_task", Integer.parseInt(cpus))
+ .put("current_working_directory", workdir)
+ .put("nodes", "1")
+ .put("required_nodes", List.of(whichnode))
+ .put("memory_per_node", new JSONObject()
+ .put("set", true)
+ .put("number", mems))
+ .put("standard_output", workdir+"/"+sifimagename+"/"+sifimagename+".out")
+ .put("standard_error", workdir+"/"+sifimagename+"/"+sifimagename+".err")
+ .put("environment",
+ new JSONArray().put("PATH=/bin:/usr/bin:/sbin:$PATH"));
+
+ String script = """
+ #!/bin/bash -e
+ find_free() { for p in $(seq 30000 31000); do ss -ltn "( sport = :$p )" | grep -q LISTEN || { echo $p; return; }; done; }
+ INNER=$(find_free)
+ PORT=%d
+ IMG="%s"
+ INTOIMAGE="cd /usr/src/app"
+ UVI="%s"
+
+ apptainer exec "$IMG" \\
+ sh -c "$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER" &
+ PID=$!
+
+ socat TCP-LISTEN:$PORT,bind=0.0.0.0,reuseaddr,fork TCP:127.0.0.1:$INNER &
+ PID_SOCAT=$!
+
+ trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT
+ wait $PID
+ """.formatted(outer, sif_location, uvicornCmd);
+
+ // ---------- 3. payload ----------
+ return new JSONObject()
+ .put("script", script)
+ .put("job", job);
+ }
+
+
+}
diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java
new file mode 100755
index 00000000..fcb0cc95
--- /dev/null
+++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java
@@ -0,0 +1,237 @@
+package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.ExecCreateCmdResponse;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.core.DockerClientBuilder;
+import okhttp3.*;
+import okhttp3.logging.HttpLoggingInterceptor;
+import org.javatuples.Tuple;
+import org.json.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+public class SlurmRest {
+ private DockerClient dockerClient = DockerClientBuilder.getInstance().build();
+ private final String RESTVERSION = "v0.0.42";
+ private final String PRE_URL;
+ private final String URL;
+ OkHttpClient httpClient = new OkHttpClient();
+
+ /**
+ *
+ * @param preUrl if slurmrestd on host preurl is http:localhost
+ */
+ public SlurmRest(String preUrl) {
+ PRE_URL = preUrl;
+ URL = PRE_URL + ":6820/slurm/" + RESTVERSION + "/";
+ }
+
+ /**
+ *
+ * @return container name -- docker container id
+ */
+ public Map containerNameID() {
+ Map nameID = new HashMap<>();
+ List exec = dockerClient.listContainersCmd().exec();
+ exec.stream().forEach((c) -> {
+ String[] names = c.getNames();// [/xxx]
+ //System.out.println(names[0]);
+ Pattern p = Pattern.compile("^/([a-zA-Z0-9]*)$");
+ Matcher m = p.matcher(names[0]);
+ String result = m.find() ? m.group(1) : "";
+ String id = c.getId();
+ nameID.put(result, id);
+ });
+ return nameID;
+ }
+
+ /**
+ *
+ * @return all containers names
+ */
+ public List listContainerNames() {
+ List exec = dockerClient.listContainersCmd().exec();
+ List containers = new ArrayList<>();
+ exec.stream().forEach(container -> {
+ containers.add(Arrays.toString(container.getNames()));
+ });
+ return containers;
+ }
+
+ /**
+ *
+ * @return check restd as container running in cluster
+ */
+ public boolean checkRESTD() {
+ List containers = listContainerNames();
+ return containers.stream().anyMatch(containerName -> containerName.contains("rest"));
+ }
+
+ /**
+ * if the administrator has registered your user with the compute node, you can use your local name,
+ * * otherwise you can only use it as root
+ * @return
+ * @throws IOException
+ */
+ public String showHostName() throws IOException {
+ ProcessBuilder pb = new ProcessBuilder();
+ pb.command("whoami");
+ Process start = pb.redirectErrorStream(true).start();
+ BufferedReader br = new BufferedReader(new InputStreamReader(start.getInputStream()));
+ return br.readLine();
+ }
+
+ /**
+ * if the administrator has registered your user with the compute node, you can use your local name, otherwise you can only use it as root
+ * @param containerName Nodes that can generate jwt tokens for users include compute and slurmctld
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String generateTokenByHost(String containerName) throws IOException, InterruptedException {
+ String hostName = showHostName();
+ String arg = "username=".concat(hostName);
+ String[] comms = new String[]{"scontrol", "token", arg};
+ // default life-time is 5min
+ String token = executeInContainer(containerName, comms);
+ String[] split = token.split("=", 2);
+ return split[1].trim();
+ }
+
+ /**
+ * if the administrator has registered your user with the compute node, you can use your local name, otherwise you can only use it as root
+ * @param containerName Nodes that can generate jwt tokens for users include compute and slurmctld
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String generateRootToken(String containerName) throws IOException, InterruptedException {
+ String arg = "username=".concat("root");
+ String[] comms = new String[]{"scontrol", "token", arg};
+ String token = executeInContainer(containerName, comms);
+ String[] split = token.split("=", 2);
+ return split[1].trim();
+ }
+
+ public String executeInContainer(String containerName, String[] commands) throws IOException, InterruptedException {
+ Map nameIDMap = containerNameID();
+ String containerId = nameIDMap.get(containerName);
+ ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(containerId)
+ .withAttachStdout(true)
+ .withAttachStderr(true)
+ .withCmd(commands)
+ .exec();
+
+ StringBuilder result = new StringBuilder();
+ dockerClient.execStartCmd(execCreateCmdResponse.getId())
+ .exec(new ResultCallback.Adapter() {
+ @Override
+ public void onNext(Frame frame) {
+ result.append(new String(frame.getPayload()));
+ }
+ }).awaitCompletion();
+ return result.toString();
+ }
+
+ /**
+ *
+ * @param token jwt password
+ * @param where Specify permitted format queries here, https://slurm.schedmd.com/rest.html
+ * @return
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String query(String token, String where) throws IOException, InterruptedException {
+ Request req = new Request.Builder().url(URL.concat(where)).header("X-SLURM-USER-TOKEN", token).get().build();
+ try (Response response = httpClient.newCall(req).execute()) {
+ return response.body().string();
+ }
+ }
+
+ /**
+ * Specify permitted format queries here, https://slurm.schedmd.com/rest.html
+ * @param token
+ * @param params
+ * @return [0]:full json response, [1]: job id
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public String[] submit(String token, JSONObject params) throws IOException, InterruptedException {
+ MediaType type = MediaType.get("application/json; charset=utf-8");
+ RequestBody requestBody = RequestBody.create(null, params.toString());
+
+ Request req = new Request.Builder().url(URL.concat("job/submit")).
+ addHeader("X-SLURM-USER-TOKEN", token).
+ addHeader("Content-Type", "application/json").
+ post(requestBody).
+ build();
+ try (Response response = httpClient.newCall(req).execute()) {
+ if (!response.isSuccessful()) {
+ throw new IOException("HTTP " + response.code() + " - " + response.message()
+ + "\n" + response.body().string());
+ }
+
+ String respStr = response.body().string();
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readTree(respStr);
+ String jobId = jsonNode.get("job_id").asText();
+ return new String[]{respStr, jobId};
+
+ }
+ }
+
+ /**
+ * cancel x-th job
+ * @param token
+ * @param jobID
+ * @return
+ * @throws IOException
+ */
+ public boolean cancelJob(String token, String jobID) throws IOException {
+ String cancelUrl = "job/".concat(jobID);
+ Request req = new Request.Builder()
+ .url(URL.concat(cancelUrl))
+ .addHeader("X-SLURM-USER-TOKEN", token)
+ .delete()
+ .build();
+ try (Response response = httpClient.newCall(req).execute()) {
+ if (!response.isSuccessful()) {
+ throw new IOException("HTTP " + response.code() + " - " + response.message()
+ + "\n" + response.body().string());
+ }
+ String respStr = response.body().string();
+ //System.out.println(respStr);
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode jsonNode = mapper.readTree(respStr);
+ if (!jsonNode.hasNonNull("error")){
+ System.out.println("[SlurmDriver] Cancel "+jobID+" Successfully");
+ return true;
+ }
+ else if(jsonNode.get("status").get("error").get("code").asInt()==2021){
+ System.out.println("[SlurmDriver] Cancel "+jobID+" Successfully");
+ return true;
+ }
+ else {
+ System.out.println("[SlurmDriver] Failed to Cancel "+jobID+" , see logs");
+ return false;
+ }
+ }
+ }
+}
+
+
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java
index cc7bb423..cf721df4 100644
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java
+++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java
@@ -7,7 +7,7 @@
import org.apache.uima.fit.factory.CollectionReaderFactory;
import org.apache.uima.fit.pipeline.SimplePipeline;
import org.dkpro.core.io.xmi.XmiWriter;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
import org.texttechnologylab.DockerUnifiedUIMAInterface.io.reader.abbyy.AbbyyDocumentReader;
import java.io.IOException;
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java
index 6f12bab0..ed1f5fe6 100644
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java
+++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java
@@ -1,130 +1,130 @@
-package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
-
-import org.apache.uima.cas.CASException;
-import org.apache.uima.fit.factory.JCasFactory;
-import org.apache.uima.jcas.JCas;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
-import org.texttechnologylab.annotation.biofid.gnfinder.MetaData;
-import org.texttechnologylab.annotation.biofid.gnfinder.Taxon;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestGNFinderLite {
- final DockerTestContainerManager container = new DockerTestContainerManager(
- "docker.texttechnologylab.org/duui-lite-gnfinder:0.1.1"
- );
-
- DUUIComposer composer;
-
- @BeforeAll
- public void setUp() throws Exception {
- composer = new DUUIComposer()
- .withLuaContext(
- new DUUILuaContext()
- .withJsonLibrary())
- .withSkipVerification(true);
- composer.addDriver(new DUUIRemoteDriver(10000));
- }
-
- @AfterAll
- public void shutdown() throws Exception {
- if (composer != null)
- composer.shutdown();
-
- container.close();
- }
-
- private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException {
- return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()));
- }
-
- private static JCas getJCas() throws ResourceInitializationException, CASException {
- JCas jCas = JCasFactory.createJCas();
- jCas.setDocumentText(
- String.join("\n", "Nach Schluß des Congresses ist eine längere Excursion vorgesehen, auf welcher die Inseln an der Küste von Pembrokshire besucht werden.", "Dieser Ausflug dürfte besonders interessant werden, weil sich hier große Brutkolonien von Puffinus puffinus und verschiedener Alcidae befinden.", "Auch Thalassidroma pelagica dürfte hier angetroffen werden.", "Bei günstigem Wetter ist ferner der Besuch einer Brutkolonie von Sula bassana vorgesehen.", "Homo sapiens sapiens."));
- jCas.setDocumentLanguage("de");
- return jCas;
- }
-
- @Test
- public void test_default() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- .build()
- );
-
- JCas jCas = getJCas();
- composer.run(jCas);
-
- printResults(jCas);
- }
-
- @Test
- public void test_with_noBayes() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- .withParameter("noBayes", "true")
- .build()
- );
-
- JCas jCas = getJCas();
- composer.run(jCas);
-
- printResults(jCas);
- }
-
- @Test
- public void test_with_allMatches() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- // Catalogue of Life and GBIF
- .withParameter("sources", "[1, 11]")
- .withParameter("allMatches", "true")
- .build()
- );
-
- JCas jCas = getJCas();
- composer.run(jCas);
-
- printResults(jCas);
- }
-
- @Test
- public void test_with_oddsDetails() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- .withParameter("oddsDetails", "true")
- .build()
- );
-
- JCas jCas = getJCas();
- composer.run(jCas);
-
- printResults(jCas);
- }
-
- private static void printResults(JCas jCas) {
- System.out.println(jCas.select(MetaData.class).findFirst().get().toString(2));
- System.out.println();
-
- for (Taxon tx : jCas.select(Taxon.class)) {
- System.out.print(tx.toString(2));
- System.out.println("\n > coveredText: \"" + tx.getCoveredText() + "\"\n");
- System.out.println();
- }
- }
-}
+//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
+//
+//import org.apache.uima.cas.CASException;
+//import org.apache.uima.fit.factory.JCasFactory;
+//import org.apache.uima.jcas.JCas;
+//import org.apache.uima.resource.ResourceInitializationException;
+//import org.junit.jupiter.api.AfterAll;
+//import org.junit.jupiter.api.BeforeAll;
+//import org.junit.jupiter.api.Test;
+//import org.junit.jupiter.api.TestInstance;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+//import org.texttechnologylab.annotation.biofid.gnfinder.MetaData;
+//import org.texttechnologylab.annotation.biofid.gnfinder.Taxon;
+//
+//import java.io.IOException;
+//import java.net.URISyntaxException;
+//
+//@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+//public class TestGNFinderLite {
+// final DockerTestContainerManager container = new DockerTestContainerManager(
+// "docker.texttechnologylab.org/duui-lite-gnfinder:0.1.1"
+// );
+//
+// DUUIComposer composer;
+//
+// @BeforeAll
+// public void setUp() throws Exception {
+// composer = new DUUIComposer()
+// .withLuaContext(
+// new DUUILuaContext()
+// .withJsonLibrary())
+// .withSkipVerification(true);
+// composer.addDriver(new DUUIRemoteDriver(10000));
+// }
+//
+// @AfterAll
+// public void shutdown() throws Exception {
+// if (composer != null)
+// composer.shutdown();
+//
+// container.close();
+// }
+//
+// private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException {
+// return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()));
+// }
+//
+// private static JCas getJCas() throws ResourceInitializationException, CASException {
+// JCas jCas = JCasFactory.createJCas();
+// jCas.setDocumentText(
+// String.join("\n", "Nach Schluß des Congresses ist eine längere Excursion vorgesehen, auf welcher die Inseln an der Küste von Pembrokshire besucht werden.", "Dieser Ausflug dürfte besonders interessant werden, weil sich hier große Brutkolonien von Puffinus puffinus und verschiedener Alcidae befinden.", "Auch Thalassidroma pelagica dürfte hier angetroffen werden.", "Bei günstigem Wetter ist ferner der Besuch einer Brutkolonie von Sula bassana vorgesehen.", "Homo sapiens sapiens."));
+// jCas.setDocumentLanguage("de");
+// return jCas;
+// }
+//
+// @Test
+// public void test_default() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+// composer.run(jCas);
+//
+// printResults(jCas);
+// }
+//
+// @Test
+// public void test_with_noBayes() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// .withParameter("noBayes", "true")
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+// composer.run(jCas);
+//
+// printResults(jCas);
+// }
+//
+// @Test
+// public void test_with_allMatches() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// // Catalogue of Life and GBIF
+// .withParameter("sources", "[1, 11]")
+// .withParameter("allMatches", "true")
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+// composer.run(jCas);
+//
+// printResults(jCas);
+// }
+//
+// @Test
+// public void test_with_oddsDetails() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// .withParameter("oddsDetails", "true")
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+// composer.run(jCas);
+//
+// printResults(jCas);
+// }
+//
+// private static void printResults(JCas jCas) {
+// System.out.println(jCas.select(MetaData.class).findFirst().get().toString(2));
+// System.out.println();
+//
+// for (Taxon tx : jCas.select(Taxon.class)) {
+// System.out.print(tx.toString(2));
+// System.out.println("\n > coveredText: \"" + tx.getCoveredText() + "\"\n");
+// System.out.println();
+// }
+// }
+//}
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java
index 12096893..1e7df5fd 100644
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java
+++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java
@@ -1,146 +1,146 @@
-package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
-
-import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.MorphologicalFeatures;
-import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS;
-import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity;
-import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma;
-import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
-import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
-import org.apache.uima.cas.CASException;
-import org.apache.uima.fit.factory.JCasFactory;
-import org.apache.uima.fit.util.JCasUtil;
-import org.apache.uima.jcas.JCas;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
-import org.texttechnologylab.annotation.SpacyAnnotatorMetaData;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestSpacyBatched {
- final DockerTestContainerManager container = new DockerTestContainerManager(
- "docker.texttechnologylab.org/duui-spacy-lua-process:0.1.0",
- 6000
- );
-
- DUUIComposer composer;
-
- @BeforeAll
- public void setUp() throws Exception {
- composer = new DUUIComposer()
- .withLuaContext(
- new DUUILuaContext()
- .withJsonLibrary())
- .withSkipVerification(true);
- composer.addDriver(new DUUIRemoteDriver(10000));
- }
-
- @AfterAll
- public void shutdown() throws Exception {
- if (composer != null)
- composer.shutdown();
-
- container.close();
- }
-
- private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException {
- return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()));
- }
-
- private static JCas getJCas() throws ResourceInitializationException, CASException {
- JCas jCas = JCasFactory.createJCas();
- jCas.setDocumentText(
- "Die Goethe Universität ist auf vier große Universitätsgelände über das Frankfurter Stadtgebiet verteilt.\n "
- + "Barack Obama war der 44. Präsident der Vereinigten Staaten von Amerika."
- );
- jCas.setDocumentLanguage("de");
- return jCas;
- }
-
- @Test
- public void test_with_sentences() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- .withParameter("spacy_model_size", "lg")
- .build()
- );
-
- JCas jCas = getJCas();
- new Sentence(jCas, 0, 104).addToIndexes();
- new Sentence(jCas, 106, 177).addToIndexes();
-
- composer.run(jCas, "lua-process-test/w-sentences");
-
- printResult(jCas);
- }
-
- @Test
- public void test_wo_sentences() throws Exception {
- composer.resetPipeline();
- composer.add(
- getComponent()
- .withParameter("spacy_model_size", "sm")
- .build()
- );
-
- JCas jCas = getJCas();
-
- composer.run(jCas, "lua-process-test/wo-sentences");
-
- printResult(jCas);
- }
-
- private static void printResult(JCas jCas) {
- System.out.println("### SpacyAnnotatorMetaData ###");
- for (SpacyAnnotatorMetaData annotation : JCasUtil.select(jCas, SpacyAnnotatorMetaData.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### Sentence ###");
- for (Sentence annotation : JCasUtil.select(jCas, Sentence.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### Token ###");
- for (Token annotation : JCasUtil.select(jCas, Token.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### Lemma ###");
- for (Lemma annotation : JCasUtil.select(jCas, Lemma.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### POS ###");
- for (POS annotation : JCasUtil.select(jCas, POS.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### MorphologicalFeatures ###");
- for (MorphologicalFeatures annotation : JCasUtil.select(jCas, MorphologicalFeatures.class)) {
- System.out.println(annotation.toString(2));
- System.out.println();
- }
-
- System.out.println("### NamedEntity ###");
- for (NamedEntity annotation : JCasUtil.select(jCas, NamedEntity.class)) {
- System.out.print(annotation.toString(2));
- System.out.printf("%n text: '%s'%n", annotation.getCoveredText());
- System.out.println();
- }
- }
-}
+//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
+//
+//import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.MorphologicalFeatures;
+//import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS;
+//import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity;
+//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma;
+//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
+//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
+//import org.apache.uima.cas.CASException;
+//import org.apache.uima.fit.factory.JCasFactory;
+//import org.apache.uima.fit.util.JCasUtil;
+//import org.apache.uima.jcas.JCas;
+//import org.apache.uima.resource.ResourceInitializationException;
+//import org.junit.jupiter.api.AfterAll;
+//import org.junit.jupiter.api.BeforeAll;
+//import org.junit.jupiter.api.Test;
+//import org.junit.jupiter.api.TestInstance;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+//import org.texttechnologylab.annotation.SpacyAnnotatorMetaData;
+//
+//import java.io.IOException;
+//import java.net.URISyntaxException;
+//
+//@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+//public class TestSpacyBatched {
+// final DockerTestContainerManager container = new DockerTestContainerManager(
+// "docker.texttechnologylab.org/duui-spacy-lua-process:0.1.0",
+// 6000
+// );
+//
+// DUUIComposer composer;
+//
+// @BeforeAll
+// public void setUp() throws Exception {
+// composer = new DUUIComposer()
+// .withLuaContext(
+// new DUUILuaContext()
+// .withJsonLibrary())
+// .withSkipVerification(true);
+// composer.addDriver(new DUUIRemoteDriver(10000));
+// }
+//
+// @AfterAll
+// public void shutdown() throws Exception {
+// if (composer != null)
+// composer.shutdown();
+//
+// container.close();
+// }
+//
+// private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException {
+// return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()));
+// }
+//
+// private static JCas getJCas() throws ResourceInitializationException, CASException {
+// JCas jCas = JCasFactory.createJCas();
+// jCas.setDocumentText(
+// "Die Goethe Universität ist auf vier große Universitätsgelände über das Frankfurter Stadtgebiet verteilt.\n "
+// + "Barack Obama war der 44. Präsident der Vereinigten Staaten von Amerika."
+// );
+// jCas.setDocumentLanguage("de");
+// return jCas;
+// }
+//
+// @Test
+// public void test_with_sentences() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// .withParameter("spacy_model_size", "lg")
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+// new Sentence(jCas, 0, 104).addToIndexes();
+// new Sentence(jCas, 106, 177).addToIndexes();
+//
+// composer.run(jCas, "lua-process-test/w-sentences");
+//
+// printResult(jCas);
+// }
+//
+// @Test
+// public void test_wo_sentences() throws Exception {
+// composer.resetPipeline();
+// composer.add(
+// getComponent()
+// .withParameter("spacy_model_size", "sm")
+// .build()
+// );
+//
+// JCas jCas = getJCas();
+//
+// composer.run(jCas, "lua-process-test/wo-sentences");
+//
+// printResult(jCas);
+// }
+//
+// private static void printResult(JCas jCas) {
+// System.out.println("### SpacyAnnotatorMetaData ###");
+// for (SpacyAnnotatorMetaData annotation : JCasUtil.select(jCas, SpacyAnnotatorMetaData.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### Sentence ###");
+// for (Sentence annotation : JCasUtil.select(jCas, Sentence.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### Token ###");
+// for (Token annotation : JCasUtil.select(jCas, Token.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### Lemma ###");
+// for (Lemma annotation : JCasUtil.select(jCas, Lemma.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### POS ###");
+// for (POS annotation : JCasUtil.select(jCas, POS.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### MorphologicalFeatures ###");
+// for (MorphologicalFeatures annotation : JCasUtil.select(jCas, MorphologicalFeatures.class)) {
+// System.out.println(annotation.toString(2));
+// System.out.println();
+// }
+//
+// System.out.println("### NamedEntity ###");
+// for (NamedEntity annotation : JCasUtil.select(jCas, NamedEntity.class)) {
+// System.out.print(annotation.toString(2));
+// System.out.printf("%n text: '%s'%n", annotation.getCoveredText());
+// System.out.println();
+// }
+// }
+//}
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java
index f3104757..a55a2bcc 100644
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java
+++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java
@@ -1,88 +1,88 @@
-package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
-
-import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
-import org.apache.uima.cas.CASException;
-import org.apache.uima.fit.factory.JCasFactory;
-import org.apache.uima.fit.util.JCasUtil;
-import org.apache.uima.jcas.JCas;
-import org.apache.uima.jcas.cas.TOP;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
-import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
-
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestSpacyEos {
- final DockerTestContainerManager container = new DockerTestContainerManager(
- "docker.texttechnologylab.org/duui-spacy-eos:0.1.0",
- 6000
- );
-
- DUUIComposer composer;
-
- @BeforeAll
- public void setUp() throws Exception {
- composer = new DUUIComposer()
- .withLuaContext(
- new DUUILuaContext()
- .withJsonLibrary())
- .withSkipVerification(true);
- composer.addDriver(new DUUIRemoteDriver(10000));
- composer.add(new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())));
- }
-
- @AfterAll
- public void shutdown() throws Exception {
- if (composer != null)
- composer.shutdown();
-
- container.close();
- }
-
- @Test
- public void testEnglish() throws Exception {
- JCas jCas = getJCas();
- jCas.setDocumentLanguage("en");
-
- composer.run(jCas, "duui-spacy-eos:0.1.0/en");
-
- printResults(jCas);
- }
-
- @NotNull
- private static JCas getJCas() throws ResourceInitializationException, CASException {
- JCas jCas = JCasFactory.createJCas();
- jCas.setDocumentText(
- "Barack Hussein Obama II (born August 4, 1961) is an American politician who was the 44th president of the United States from 2009 to 2017. "
- + "A member of the Democratic Party, he was the first African-American president in American history. "
- + "Obama previously served as a U.S. senator representing Illinois from 2005 to 2008 and as an Illinois state senator from 1997 to 2004."
- );
- return jCas;
- }
-
- @Test
- public void testNoLang() throws Exception {
- JCas jCas = getJCas();
-
- composer.run(jCas, "duui-spacy-eos:0.1.0/no-lang");
-
- printResults(jCas);
- }
-
- private static void printResults(JCas jCas) {
- for (TOP annotation : JCasUtil.select(jCas, TOP.class)) {
- System.out.print(annotation.toString(2));
- if (annotation instanceof Sentence) {
- System.out.printf("%n text: '%s'%n%n", ((Sentence) annotation).getCoveredText());
- } else {
- System.out.printf("%n%n");
- }
- }
- }
-}
\ No newline at end of file
+//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process;
+//
+//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
+//import org.apache.uima.cas.CASException;
+//import org.apache.uima.fit.factory.JCasFactory;
+//import org.apache.uima.fit.util.JCasUtil;
+//import org.apache.uima.jcas.JCas;
+//import org.apache.uima.jcas.cas.TOP;
+//import org.apache.uima.resource.ResourceInitializationException;
+//import org.jetbrains.annotations.NotNull;
+//import org.junit.jupiter.api.AfterAll;
+//import org.junit.jupiter.api.BeforeAll;
+//import org.junit.jupiter.api.Test;
+//import org.junit.jupiter.api.TestInstance;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver;
+//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+//
+//@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+//public class TestSpacyEos {
+// final DockerTestContainerManager container = new DockerTestContainerManager(
+// "docker.texttechnologylab.org/duui-spacy-eos:0.1.0",
+// 6000
+// );
+//
+// DUUIComposer composer;
+//
+// @BeforeAll
+// public void setUp() throws Exception {
+// composer = new DUUIComposer()
+// .withLuaContext(
+// new DUUILuaContext()
+// .withJsonLibrary())
+// .withSkipVerification(true);
+// composer.addDriver(new DUUIRemoteDriver(10000));
+// composer.add(new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())));
+// }
+//
+// @AfterAll
+// public void shutdown() throws Exception {
+// if (composer != null)
+// composer.shutdown();
+//
+// container.close();
+// }
+//
+// @Test
+// public void testEnglish() throws Exception {
+// JCas jCas = getJCas();
+// jCas.setDocumentLanguage("en");
+//
+// composer.run(jCas, "duui-spacy-eos:0.1.0/en");
+//
+// printResults(jCas);
+// }
+//
+// @NotNull
+// private static JCas getJCas() throws ResourceInitializationException, CASException {
+// JCas jCas = JCasFactory.createJCas();
+// jCas.setDocumentText(
+// "Barack Hussein Obama II (born August 4, 1961) is an American politician who was the 44th president of the United States from 2009 to 2017. "
+// + "A member of the Democratic Party, he was the first African-American president in American history. "
+// + "Obama previously served as a U.S. senator representing Illinois from 2005 to 2008 and as an Illinois state senator from 1997 to 2004."
+// );
+// return jCas;
+// }
+//
+// @Test
+// public void testNoLang() throws Exception {
+// JCas jCas = getJCas();
+//
+// composer.run(jCas, "duui-spacy-eos:0.1.0/no-lang");
+//
+// printResults(jCas);
+// }
+//
+// private static void printResults(JCas jCas) {
+// for (TOP annotation : JCasUtil.select(jCas, TOP.class)) {
+// System.out.print(annotation.toString(2));
+// if (annotation instanceof Sentence) {
+// System.out.printf("%n text: '%s'%n%n", ((Sentence) annotation).getCoveredText());
+// } else {
+// System.out.printf("%n%n");
+// }
+// }
+// }
+//}
\ No newline at end of file
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java
deleted file mode 100644
index ad03e7dc..00000000
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java
+++ /dev/null
@@ -1,95 +0,0 @@
-package org.texttechnologylab.DockerUnifiedUIMAInterface.test;
-
-import com.github.dockerjava.api.DockerClient;
-import com.github.dockerjava.api.command.CreateContainerResponse;
-import com.github.dockerjava.api.model.ExposedPort;
-import com.github.dockerjava.api.model.HostConfig;
-import com.github.dockerjava.api.model.PortBinding;
-import com.github.dockerjava.core.DockerClientBuilder;
-import org.apache.commons.lang.math.IntRange;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-/**
- * Test helper class to create and run a container image for testing purposes.
- * Will search for a random free port
- */
-public class DockerTestContainerManager implements AutoCloseable {
- final DockerClient dockerClient = DockerClientBuilder.getInstance().build();
- final CreateContainerResponse container;
- final int port = searchFreePort();
-
- public DockerTestContainerManager(String imageName) {
- this(imageName, "test-duui-" + UUID.randomUUID(), 5000);
- }
-
- public DockerTestContainerManager(String imageName, String containerName) {
- this(imageName, containerName, 5000);
- }
-
- public DockerTestContainerManager(String imageName, long startupDelay) {
- this(imageName, "test-duui-" + UUID.randomUUID(), startupDelay);
- }
-
- public DockerTestContainerManager(String imageName, String containerName, long startupDelay) {
- System.out.printf("[DockerTestContainerManager] Creating Container Image for %s%n", imageName);
- container = dockerClient.createContainerCmd(imageName)
- .withHostConfig(
- HostConfig.newHostConfig()
- .withAutoRemove(true)
- .withPublishAllPorts(true)
- .withPortBindings(PortBinding.parse("%d:9714".formatted(port)))
- )
- .withExposedPorts(ExposedPort.tcp(9714))
- .withName(containerName)
- .exec();
- System.out.printf("[DockerTestContainerManager] Container Image Created: %s%n", container.getId());
- System.out.printf("[DockerTestContainerManager] Starting Container as %s with port binding %d:9714%n", containerName, port);
- dockerClient.startContainerCmd(container.getId()).exec();
-
- try {
- System.out.printf("[DockerTestContainerManager] Waiting %dms for Container to Come Online%n", startupDelay);
- Thread.sleep(startupDelay);
- } catch (InterruptedException e) {
- }
-
- System.out.println("[DockerTestContainerManager] Container Started: " + container.getId());
- }
-
- private static int searchFreePort() {
- List ports = Arrays.stream(new IntRange(30000, 40000).toArray()).boxed().collect(Collectors.toList());
- Collections.shuffle(ports);
- for (int port : ports) {
- try (ServerSocket socket = new ServerSocket(port)) {
- if (socket.getLocalPort() == port) {
- return port;
- }
- } catch (IOException e) {
- if (!e.getMessage().contains("Address already in use")) {
- throw new RuntimeException(e);
- }
- }
- }
- throw new RuntimeException("No free port found");
- }
-
- public int getPort() {
- return port;
- }
-
- @Override
- public void close() throws Exception {
- System.out.println("[DockerTestContainerManager] Stopping Container: " + container.getId());
- dockerClient.stopContainerCmd(container.getId()).withTimeout(10).exec();
- System.out.println("[DockerTestContainerManager] Container Stopped");
- System.out.println("[DockerTestContainerManager] Stopping Docker Client");
- dockerClient.close();
- }
-
-}
\ No newline at end of file
diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java
index 7333847b..1dfdce5d 100644
--- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java
+++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java
@@ -86,7 +86,7 @@ static void resetCas() {
}
@AfterAll
- static void afterAll() throws UnknownHostException {
+ static void afterAll() throws IOException, InterruptedException {
composer.shutdown();
}
diff --git a/src/test/java/slurmDriver/ParlbertTopicGermanTest.java b/src/test/java/slurmDriver/ParlbertTopicGermanTest.java
new file mode 100755
index 00000000..19314c9d
--- /dev/null
+++ b/src/test/java/slurmDriver/ParlbertTopicGermanTest.java
@@ -0,0 +1,147 @@
+package slurmDriver;
+
+import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS;
+import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity;
+import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
+import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
+import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.fit.util.JCasUtil;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.util.XmlCasSerializer;
+import org.dkpro.core.io.xmi.XmiWriter;
+import org.junit.jupiter.api.Test;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.*;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.io.AsyncCollectionReader;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+import org.hucompute.textimager.uima.type.category.CategoryCoveredTagged;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+
+public class ParlbertTopicGermanTest {
+
+ public static final String dijkstraExampleText = "Der Algorithmus von Dijkstra (nach seinem Erfinder Edsger W. Dijkstra) ist ein Algorithmus aus der Klasse der Greedy-Algorithmen[1] und löst das Problem der kürzesten Pfade für einen gegebenen Startknoten. " +
+ "Er berechnet somit einen kürzesten Pfad zwischen dem gegebenen Startknoten und einem der (oder allen) übrigen Knoten in einem kantengewichteten Graphen (sofern dieser keine Negativkanten enthält)." +
+ "Für unzusammenhängende ungerichtete Graphen ist der Abstand zu denjenigen Knoten unendlich, zu denen kein Pfad vom Startknoten aus existiert. Dasselbe gilt auch für gerichtete nicht stark zusammenhängende Graphen. Dabei wird der Abstand synonym auch als Entfernung, Kosten oder Gewicht bezeichnet.";
+
+ @Test
+ public void test() throws Exception {
+
+ int iWorkers = 1;
+ DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary();
+
+ // Instanziierung des Composers, mit einigen Parametern
+ DUUIComposer composer = new DUUIComposer()
+ .withSkipVerification(true) // wir überspringen die Verifikation aller Componenten =)
+ .withLuaContext(ctx) // wir setzen den definierten Kontext
+ .withWorkers(iWorkers); // wir geben dem Composer eine Anzahl an Threads mit.
+
+
+ DUUIDockerDriver docker_driver = new DUUIDockerDriver();
+ DUUIRemoteDriver remoteDriver = new DUUIRemoteDriver(1000);
+ DUUIUIMADriver uima_driver = new DUUIUIMADriver()
+ .withDebug(false);
+ composer.addDriver(docker_driver, remoteDriver,uima_driver);
+
+ boolean useDockerImage = false;
+ if (useDockerImage){
+ composer.add(new DUUIDockerDriver.Component("docker.texttechnologylab.org/parlbert-topic-german:latest")
+ .withScale(iWorkers)
+ .build());
+ }else{
+ composer.add(new DUUIRemoteDriver.Component("http://localhost:20000")
+ .withScale(iWorkers)
+ .build());
+ composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/slurm",
+ XmiWriter.PARAM_PRETTY_PRINT, true,
+ XmiWriter.PARAM_OVERWRITE, true,
+ XmiWriter.PARAM_VERSION, "1.1"
+ )).build());
+ }
+
+ // Create basic test jCas.
+ JCas jCas = JCasFactory.createText(dijkstraExampleText, "de");
+
+ new Sentence(jCas, 0, 206).addToIndexes();
+ new Sentence(jCas, 206, 402).addToIndexes();
+ new Sentence(jCas, 402, 544).addToIndexes();
+ new Sentence(jCas, 544, 616).addToIndexes();
+ new Sentence(jCas, 616, 699).addToIndexes();
+
+ composer.run(jCas, "test");
+
+ // Print Result
+ Collection categoryCoveredTaggeds = JCasUtil.select(jCas, CategoryCoveredTagged.class).stream().sorted((c1, c2) -> c1.getBegin()-c2.getBegin()).collect(Collectors.toList());
+ for(CategoryCoveredTagged categoryCoveredTagged: categoryCoveredTaggeds){
+ System.out.println(categoryCoveredTagged.getBegin() + " - " + categoryCoveredTagged.getEnd() + " " + categoryCoveredTagged.getValue() + ": " + categoryCoveredTagged.getScore());
+ }
+
+
+
+ }
+
+ @Test
+ public void singleTokenizedTestDe() throws Exception {
+ DUUIComposer composer = new DUUIComposer()
+ .withLuaContext(
+ new DUUILuaContext()
+ .withJsonLibrary()
+ )
+ .withSkipVerification(true);
+
+ DUUIRemoteDriver remote_driver = new DUUIRemoteDriver(10000);
+ DUUIUIMADriver uima_driver = new DUUIUIMADriver();
+ composer.addDriver(remote_driver,uima_driver);
+
+ composer.add(
+ new DUUIRemoteDriver.Component("http://127.0.0.1:20000")
+ .withParameter("use_existing_tokens", String.valueOf(true))
+ );
+ composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/slurm",
+ XmiWriter.PARAM_PRETTY_PRINT, true,
+ XmiWriter.PARAM_OVERWRITE, true,
+ XmiWriter.PARAM_VERSION, "1.1"
+ )).build());
+
+ String language = "de";
+
+ JCas cas = JCasFactory.createJCas();
+ cas.setDocumentText("Das ist ein IPhone von Apple. Und das ist ein iMac.");
+ cas.setDocumentLanguage(language);
+
+ int[][] tokens1 = new int[][]{
+ new int[]{0, 3}, //Das
+ new int[]{4, 7}, //ist
+ new int[]{8, 11}, //ein
+ new int[]{12, 18}, //IPhone
+ new int[]{19, 22}, //von
+ new int[]{23, 28}, //Apple
+ new int[]{28, 29}, //.
+ new int[]{30, 33}, //Und
+ new int[]{34, 37}, //das
+ new int[]{38, 41}, //ist
+ new int[]{42, 45}, //ein
+ new int[]{46, 50}, //iMac
+ new int[]{50, 51} //.
+ };
+
+ for (int[] tokenPos : tokens1) {
+ Token token = new Token(cas, tokenPos[0], tokenPos[1]);
+ token.addToIndexes();
+ }
+
+ composer.run(cas);
+
+ }
+}
diff --git a/src/test/java/slurmDriver/RestApi.java b/src/test/java/slurmDriver/RestApi.java
new file mode 100755
index 00000000..77a681be
--- /dev/null
+++ b/src/test/java/slurmDriver/RestApi.java
@@ -0,0 +1,348 @@
+package slurmDriver;
+
+import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.uima.UIMAException;
+import org.apache.uima.cas.CASException;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.jcas.JCas;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.dkpro.core.io.xmi.XmiWriter;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.SlurmUtils;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaSandbox;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
+
+public class RestApi {
+ SlurmRest slurmRest = new SlurmRest("http:localhost");
+
+ @Test
+ public void dockerCommandTest() throws IOException {
+ boolean b = slurmRest.checkRESTD();
+ System.out.println(b);
+ System.out.println(slurmRest.showHostName());
+ List strings = slurmRest.listContainerNames();
+ System.out.println(strings);
+
+ }
+
+ @Test
+ public void seeInfos() throws IOException, InterruptedException {
+ // GET /slurm/v0.0.42/diag
+ // GET /slurm/v0.0.42/nodes
+ // GET /slurm/v0.0.42/jobs
+
+ String s = slurmRest.generateRootToken("compute1");
+ System.out.println("=============================================================================");
+ System.out.println(slurmRest.query(s, "diag"));
+ System.out.println("=============================================================================");
+ System.out.println(slurmRest.query(s, "jobs"));
+ System.out.println("=============================================================================");
+
+
+ }
+
+ @Test
+ public void splitToken() {
+ String token = "SLURM_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NTE4ODY3MzIsImlhdCI6MTc1MTg4NDkzMiwic3VuIjoiamQifQ.uzplyXbsDPPix1hEU7JNcUhWUm8YPs6UHNTUpODLyE0";
+ String[] split = token.split("=", 2);
+ split[0] = split[0].trim();
+ split[1] = split[1].trim();
+ System.out.println(split[0]);
+ System.out.println(split[1]);
+ }
+
+ @Test
+ public void submitByRoot() throws IOException, InterruptedException {
+ JSONObject job = new JSONObject()
+ .put("name", "rest_smoke")
+ .put("partition", "normal")
+ .put("time_limit", 300)
+ .put("current_working_directory", "/data")
+ .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/"));
+
+ JSONObject payload = new JSONObject()
+ .put("script", "#!/bin/bash\nsrun hostname")
+ .put("job", job);
+
+ String token = slurmRest.generateRootToken("compute1");
+ Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println);
+ }
+
+ @Test
+ public void seeJobIDn() throws IOException, InterruptedException {
+ String s = slurmRest.generateRootToken("compute1");
+ System.out.println("=============================================================================");
+ System.out.println(slurmRest.query(s, "job/13"));
+ }
+
+ @Test
+ public void submitSpacy() throws IOException, InterruptedException {
+
+ JSONObject job = new JSONObject()
+ .put("name", "spacy_test")
+ .put("partition", "normal")
+ .put("cpus_per_task", 2)
+ .put("required_nodes", List.of("compute1"))
+ .put( "tres_per_node", "gres/gpu=1")
+ .put("memory_per_node", new JSONObject()
+ .put("set", true)
+ .put("number", 2048))
+ .put("time_limit", 600)//minute
+ .put("current_working_directory", "/data")
+ .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/"));
+
+ JSONObject payload = new JSONObject()
+ .put("script",
+ "#!/bin/bash\n" +
+ "PORT=20000 \n" +
+ "INNER=9714 \n" +
+ "IMG=\"/sif_pool/spacy.sif\" " +
+ "INTOIMAGE=\"cd /usr/src/app\"\n" +
+ "UVI=\"uvicorn textimager_duui_spacy:app\" \n" +
+ "\n" +
+ "apptainer exec \"$IMG\" \\\n" +
+ " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &\n" +
+ "\n" +
+ "PID=$!\n" +
+ "\n" +
+ "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\n" +
+ "\n" +
+ "PID_SOCAT=$!\n" +
+ "\n" +
+ "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\n" +
+ "\n" +
+ "wait $PID")
+ .put("job", job);
+
+ String token = slurmRest.generateRootToken("compute1");
+ Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println);
+ }
+
+ @Test
+ public void cancelJob() throws IOException, InterruptedException {
+ String token = slurmRest.generateRootToken("compute1");
+ assert slurmRest.cancelJob(token, "5");
+
+ }
+
+ @Test
+ public void runParlbert() throws IOException, InterruptedException {
+ JSONObject job = new JSONObject()
+ .put("name", "cuda_test")
+ .put("partition", "normal")
+ .put("time_limit", 600)
+ .put("required_nodes", "compute1")
+ .put("cpus_per_task", 2)
+ .put("current_working_directory", "/data")
+ .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/"));
+
+ JSONObject payload = new JSONObject()
+ .put("script",
+ "#!/bin/bash\n" +
+ "#SBATCH --gres=gpu:1\n" +
+ "#SBATCH --partition=normal\n" +
+ "#SBATCH --nodelist=compute1\n" +
+
+ "PORT=20000 \n" +
+ "INNER=9714 \n" +
+ "IMG=\"/sif_pool/bert_cuda.sif\" " +
+ "INTOIMAGE=\"cd /usr/src/app\"\n" +
+ "UVI=\"uvicorn parlbert_topic_german:app\" \n" +
+ "\n" +
+ "apptainer exec --nv \"$IMG\" \\\n" +
+ " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &\n" +
+ "\n" +
+ "PID=$!\n" +
+ "\n" +
+ "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\n" +
+ "\n" +
+ "PID_SOCAT=$!\n" +
+ "\n" +
+ "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\n" +
+ "\n" +
+ "wait $PID")
+ .put("job", job);
+ System.out.println(payload.toString());
+ //String token = slurmRest.generateRootToken("compute1");
+ //Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println);
+ }
+
+ @Test
+ public void runNvidiasmi() throws IOException, InterruptedException {
+ JSONObject job = new JSONObject()
+ .put("name", "smi_test")
+ .put("partition", "normal")
+ .put("time_limit", 600)
+ .put("current_working_directory", "/data")
+ .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/"));
+
+ JSONObject payload = new JSONObject()
+ .put("script",
+ "#!/bin/bash\n#SBATCH --gres=gpu:1\nnvidia-smi")
+ .put("job", job);
+
+ String token = slurmRest.generateRootToken("compute1");
+ Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println);
+ }
+
+
+ @Test
+ public void pipelineTest_json() throws Exception {
+ SlurmRest rest = new SlurmRest("http://localhost");
+ String token = rest.generateRootToken("compute1");
+
+ int iWorkers = 1; //
+ JCas jc = JCasFactory.createText("Early life\n" +
+ "William Gordon Gordon-Cumming was born on 20 July 1848 at Sanquhar House, near Forres, Morayshire.[1] His parents were Alexander Penrose Gordon-Cumming—the third of the Gordon-Cumming baronets—and Anne Pitcairn (née Campbell). William was the second of the couple's four children and their eldest son. His uncle, Roualeyn George Gordon-Cumming, was a noted big-game hunter; and his aunt, Constance Gordon-Cumming, was a travel writer. Gordon-Cumming was educated at the English boarding schools Eton and Wellington.[1][2]\n" +
+ "\n" +
+ "At the age of eighteen he inherited the baronetcy and became chief of Clan Cumming; his line had been traced from the fourth century, through Charlemagne. His inheritance included three Morayshire estates: Altyre near Forres, Gordonstoun near Elgin, and the village of Dallas. The estates totalled 38,500 acres (156 km2) of poor quality land;[1][3] the annual income from the estates in around 1890 has been given as either £60,000[4] or £80,000.[5][a]\n" +
+ "\n" +
+ "Military career\n" +
+ "Although Gordon-Cumming had asthma and was blind in one eye, he purchased a commission as ensign in the Scots Fusilier Guards (renamed in 1877 as the Scots Guards) in 1868 (dated from 25 December 1867).[1][7][8] He was promoted to regimental lieutenant and to the brevet rank of captain in the army by purchase on 17 May 1871, the last year in which commissions were available for purchase.[9][b] He volunteered for service in South Africa in the Anglo-Zulu War, where he served gallantly and was mentioned in despatches; he was the first man to enter Cetshwayo's kraal after the Battle of Ulundi (1879). That year he conveyed the condolences of the army to the ex-Empress Eugénie on the death of her son, Napoléon, Prince Imperial.[1][2]\n" +
+ "\n" +
+ "Gordon-Cumming was promoted to the regimental rank of captain and the army rank of lieutenant-colonel on 28 July 1880.[11] He served in Egypt during the Anglo-Egyptian War (1882) and in the Sudan in the Mahdist War (1884–1885), the last of which was with the Guards Camel Regiment in the Desert Column.[2][4][c] He was promoted to the regimental rank of major on 23 May 1888.[13]\n" +
+ "\n" +
+ "He also found time for independent travel and adventure, stalking tigers on foot in India and hunting in the Rocky Mountains in the US;[1][4] in 1871 he published an account of his travels in India, Wild Men and Wild Beasts: Scenes in Camp and Jungle.[14] The work covers the best routes to travel to and from India and which animals are available for hunting in which season, as well as the equipment a hunter would need to take on an expedition.[15] He concluded his work with the following:\n" +
+ "\n" +
+ "The record of my doings might no doubt have been more acceptable to the general reader had it been more varied with matter other than mere slaughter, and had the tale of bloodshed been more frequently relieved by accounts of the geography, scenery, and natural history, human and bestial, of the country; but all these have been well described elsewhere, and by abler pens.[16]\n" +
+ "\n" +
+ "Royal baccarat scandal\n" +
+ "Main article: Royal baccarat scandal\n" +
+ "In September 1890 Arthur Wilson, the 52-year-old Hull-based owner of a shipping business, invited Gordon-Cumming, along with Edward, Prince of Wales, to a house party at Tranby Croft in the East Riding of Yorkshire;[17][18] Gordon-Cumming and the prince had been friends for over twenty years.[19] Among the other people present that weekend were Wilson's wife, Mary, their son, Stanley, their daughter, Ethel, and her husband, Edward Lycett Green, who was the son of Sir Edward Green, 1st Baronet, a local Conservative politician.[20] Several members of the prince's inner circle were also invited to stay, including Sir Christopher Sykes—the Conservative MP for Beverley—the equerry Tyrwhitt Wilson, Lord Coventry, Lord Edward Somerset, Captain Arthur Somerset—his cousin—and Lieutenant-General Owen Williams, along with their wives. Also accompanying the party was Lieutenant Berkeley Levett, a brother officer to Gordon-Cumming in the Scots Guards and a friend of the Wilson family.[21]\n" +
+ "\n" +
+ "During the evenings of the weekend, Edward insisted on playing baccarat, a game that was at the time illegal if gambling was involved;[22] many of the house joined in, including Gordon-Cumming, Levett and Stanley Wilson. The prince acted as the dealer.[23][d] On the first night of play, Stanley Wilson thought he saw Gordon-Cumming add two red £5 counters onto his stake after the hand had finished, but before the winnings had been paid, thus increasing the money paid to him by the bank—a method of cheating known in casinos as la poussette. He alerted Levett, sitting next to him, and both men thought they saw Gordon-Cumming repeat the act on the next hand.[24][25]\n" +
+ "\n" +
+ "After the second evening of play Lycett Green, Stanley Wilson and Arthur and Edward Somerset confronted Gordon-Cumming and accused him of cheating.[18][e] Gordon-Cumming insisted they had been mistaken, and explained that he played the coup de trois system of betting,[f] in which if he won a hand with a £5 stake, he would add his winnings to the stake, together with another £5, as the stake for the next hand.[28][g] Edward, after hearing from his advisors and the accusers, believed what they had told him.[29] In order to avoid a scandal involving the prince, Gordon-Cumming gave way to pressure from the attendant royal courtiers to sign a statement undertaking never to play cards again in return for a pledge that no-one present would speak of the incident to anyone else.[30][31]\n" +
+ "\n" +
+ "In consideration of the promise made by the gentlemen whose names are subscribed to preserve my silence with reference to an accusation which has been made in regard to my conduct at baccarat on the nights of Monday and Tuesday the 8th and 9th at Tranby Croft, I will on my part solemnly undertake never to play cards again as long as I live.\n" +
+ "\n" +
+ "— (Signed) W. Gordon-Cumming[32]");
+ DocumentMetaData dmd = new DocumentMetaData(jc);
+ dmd.setDocumentId("test_rest");
+ dmd.setDocumentTitle("test_REST");
+ dmd.addToIndexes();
+ DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary();
+ DUUILuaSandbox sandbox = new DUUILuaSandbox();
+ sandbox._allowAllJavaClasses = true;
+ ctx.withSandbox(sandbox);
+ DUUIComposer composer = new DUUIComposer()
+ .withSkipVerification(true)
+ .withLuaContext(ctx)
+ .withWorkers(iWorkers);
+ DUUIUIMADriver uimaDriver = new DUUIUIMADriver();
+ DUUISlurmDriver slurmDriver = new DUUISlurmDriver(rest);
+
+ composer.addDriver(uimaDriver, slurmDriver);
+
+ composer.resetPipeline();
+ DUUIPipelineComponent com1 = new DUUISlurmDriver.Component(
+ new DUUIPipelineComponent().withSlurmScript(
+ "{\"job\":{\"environment\":[\"PATH=/bin/:/usr/bin/:/sbin/\"],\"partition\":\"normal\",\"time_limit\":600,\"name\":\"spacy_test\",\"current_working_directory\":\"/data\"},\"script\":\"#!/bin/bash\\n \\nPORT=20000 \\nINNER=9714 \\nIMG=\\\"/sif_pool/spacy.sif\\\" INTOIMAGE=\\\"cd /usr/src/app\\\"\\nUVI=\\\"uvicorn textimager_duui_spacy:app\\\" \\n\\napptainer exec --nv \\\"$IMG\\\" \\\\\\n sh -c \\\"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\\\" &\\n\\nPID=$!\\n\\nsocat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\\n\\nPID_SOCAT=$!\\n\\ntrap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\\n\\nwait $PID\"}\n"
+ ).withSlurmHostPort("20000")).build();
+
+ System.out.println(com1.getSlurmScript());
+
+ composer.add(com1);
+ composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/",
+ XmiWriter.PARAM_PRETTY_PRINT, true,
+ XmiWriter.PARAM_OVERWRITE, true,
+ XmiWriter.PARAM_VERSION, "1.1",
+ XmiWriter.PARAM_COMPRESSION, "GZIP"))
+ .build());
+
+ composer.run(jc);
+
+
+ }
+
+ @Test
+ public void pipelineTest_parameters() throws Exception {
+ SlurmRest rest = new SlurmRest("http://localhost");
+
+
+ int iWorkers = 2; //
+ JCas jc = JCasFactory.createText("Early life\n" +
+ "William Gordon Gordon-Cumming was born on 20 July 1848 at Sanquhar House, near Forres, Morayshire.[1] His parents were Alexander Penrose Gordon-Cumming—the third of the Gordon-Cumming baronets—and Anne Pitcairn (née Campbell). William was the second of the couple's four children and their eldest son. His uncle, Roualeyn George Gordon-Cumming, was a noted big-game hunter; and his aunt, Constance Gordon-Cumming, was a travel writer. Gordon-Cumming was educated at the English boarding schools Eton and Wellington.[1][2]\n" +
+ "\n" +
+ "At the age of eighteen he inherited the baronetcy and became chief of Clan Cumming; his line had been traced from the fourth century, through Charlemagne. His inheritance included three Morayshire estates: Altyre near Forres, Gordonstoun near Elgin, and the village of Dallas. The estates totalled 38,500 acres (156 km2) of poor quality land;[1][3] the annual income from the estates in around 1890 has been given as either £60,000[4] or £80,000.[5][a]\n" +
+ "\n" +
+ "Military career\n" +
+ "Although Gordon-Cumming had asthma and was blind in one eye, he purchased a commission as ensign in the Scots Fusilier Guards (renamed in 1877 as the Scots Guards) in 1868 (dated from 25 December 1867).[1][7][8] He was promoted to regimental lieutenant and to the brevet rank of captain in the army by purchase on 17 May 1871, the last year in which commissions were available for purchase.[9][b] He volunteered for service in South Africa in the Anglo-Zulu War, where he served gallantly and was mentioned in despatches; he was the first man to enter Cetshwayo's kraal after the Battle of Ulundi (1879). That year he conveyed the condolences of the army to the ex-Empress Eugénie on the death of her son, Napoléon, Prince Imperial.[1][2]\n" +
+ "\n" +
+ "Gordon-Cumming was promoted to the regimental rank of captain and the army rank of lieutenant-colonel on 28 July 1880.[11] He served in Egypt during the Anglo-Egyptian War (1882) and in the Sudan in the Mahdist War (1884–1885), the last of which was with the Guards Camel Regiment in the Desert Column.[2][4][c] He was promoted to the regimental rank of major on 23 May 1888.[13]\n" +
+ "\n" +
+ "He also found time for independent travel and adventure, stalking tigers on foot in India and hunting in the Rocky Mountains in the US;[1][4] in 1871 he published an account of his travels in India, Wild Men and Wild Beasts: Scenes in Camp and Jungle.[14] The work covers the best routes to travel to and from India and which animals are available for hunting in which season, as well as the equipment a hunter would need to take on an expedition.[15] He concluded his work with the following:\n" +
+ "\n" +
+ "The record of my doings might no doubt have been more acceptable to the general reader had it been more varied with matter other than mere slaughter, and had the tale of bloodshed been more frequently relieved by accounts of the geography, scenery, and natural history, human and bestial, of the country; but all these have been well described elsewhere, and by abler pens.[16]\n" +
+ "\n" +
+ "Royal baccarat scandal\n" +
+ "Main article: Royal baccarat scandal\n" +
+ "In September 1890 Arthur Wilson, the 52-year-old Hull-based owner of a shipping business, invited Gordon-Cumming, along with Edward, Prince of Wales, to a house party at Tranby Croft in the East Riding of Yorkshire;[17][18] Gordon-Cumming and the prince had been friends for over twenty years.[19] Among the other people present that weekend were Wilson's wife, Mary, their son, Stanley, their daughter, Ethel, and her husband, Edward Lycett Green, who was the son of Sir Edward Green, 1st Baronet, a local Conservative politician.[20] Several members of the prince's inner circle were also invited to stay, including Sir Christopher Sykes—the Conservative MP for Beverley—the equerry Tyrwhitt Wilson, Lord Coventry, Lord Edward Somerset, Captain Arthur Somerset—his cousin—and Lieutenant-General Owen Williams, along with their wives. Also accompanying the party was Lieutenant Berkeley Levett, a brother officer to Gordon-Cumming in the Scots Guards and a friend of the Wilson family.[21]\n" +
+ "\n" +
+ "During the evenings of the weekend, Edward insisted on playing baccarat, a game that was at the time illegal if gambling was involved;[22] many of the house joined in, including Gordon-Cumming, Levett and Stanley Wilson. The prince acted as the dealer.[23][d] On the first night of play, Stanley Wilson thought he saw Gordon-Cumming add two red £5 counters onto his stake after the hand had finished, but before the winnings had been paid, thus increasing the money paid to him by the bank—a method of cheating known in casinos as la poussette. He alerted Levett, sitting next to him, and both men thought they saw Gordon-Cumming repeat the act on the next hand.[24][25]\n" +
+ "\n" +
+ "After the second evening of play Lycett Green, Stanley Wilson and Arthur and Edward Somerset confronted Gordon-Cumming and accused him of cheating.[18][e] Gordon-Cumming insisted they had been mistaken, and explained that he played the coup de trois system of betting,[f] in which if he won a hand with a £5 stake, he would add his winnings to the stake, together with another £5, as the stake for the next hand.[28][g] Edward, after hearing from his advisors and the accusers, believed what they had told him.[29] In order to avoid a scandal involving the prince, Gordon-Cumming gave way to pressure from the attendant royal courtiers to sign a statement undertaking never to play cards again in return for a pledge that no-one present would speak of the incident to anyone else.[30][31]\n" +
+ "\n" +
+ "In consideration of the promise made by the gentlemen whose names are subscribed to preserve my silence with reference to an accusation which has been made in regard to my conduct at baccarat on the nights of Monday and Tuesday the 8th and 9th at Tranby Croft, I will on my part solemnly undertake never to play cards again as long as I live.\n" +
+ "\n" +
+ "— (Signed) W. Gordon-Cumming[32]");
+ DocumentMetaData dmd = new DocumentMetaData(jc);
+ dmd.setDocumentId("test_rest");
+ dmd.setDocumentTitle("test_REST");
+ dmd.addToIndexes();
+ DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary();
+ DUUILuaSandbox sandbox = new DUUILuaSandbox();
+ sandbox._allowAllJavaClasses = true;
+ ctx.withSandbox(sandbox);
+ DUUIComposer composer = new DUUIComposer()
+ .withSkipVerification(true)
+ .withLuaContext(ctx)
+ .withWorkers(iWorkers);
+ DUUIUIMADriver uimaDriver = new DUUIUIMADriver();
+ DUUISlurmDriver slurmDriver = new DUUISlurmDriver(rest);
+
+ composer.addDriver(uimaDriver, slurmDriver);
+
+ composer.resetPipeline();
+ DUUIPipelineComponent com1 = new DUUISlurmDriver.Component(
+ new DUUIPipelineComponent().
+ withSlurmPartition("normal").
+ withSlurmNodelist("compute1").
+ withSlurmWorkDir("/data").
+ withSlurmUvicorn("uvicorn textimager_duui_spacy:app").
+ withSlurmSIFName("spacy").
+ withSlurmMemory("1024").
+ withSlurmRuntime("5").withSlurmCPUs("2").
+ withSlurmSaveIn("/sif_pool/spacy.sif").withSlurmJobName("spacy").withSlurmCPUs("2").
+ withSlurmGPU("0").withScale(2)
+ ).build();
+ composer.add(com1);
+ composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/",
+ XmiWriter.PARAM_PRETTY_PRINT, true,
+ XmiWriter.PARAM_OVERWRITE, true,
+ XmiWriter.PARAM_VERSION, "1.1",
+ XmiWriter.PARAM_COMPRESSION, "GZIP"))
+ .build());
+ composer.run(jc);
+
+ }
+
+}
diff --git a/src/test/java/slurmDriver/universTest.java b/src/test/java/slurmDriver/universTest.java
new file mode 100755
index 00000000..777b4c8c
--- /dev/null
+++ b/src/test/java/slurmDriver/universTest.java
@@ -0,0 +1,96 @@
+package slurmDriver;
+
+import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.jcas.JCas;
+import org.dkpro.core.io.xmi.XmiWriter;
+import org.junit.jupiter.api.Test;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIDockerDriver;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.SlurmUtils;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest;
+import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.*;
+
+import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
+
+public class universTest {
+ @Test
+ public void universTest() throws Exception {
+ int iWorkers = 2; // define the number of workers
+
+ JCas jc = JCasFactory.createJCas(); // An empty CAS document is defined.
+
+// load content into jc ...
+
+// Defining LUA-Context for communication
+ DUUILuaContext ctx = LuaConsts.getJSON();
+// The composer is defined and initialized with a standard Lua context as well with a storage backend.
+ DUUIComposer composer = new DUUIComposer().withLuaContext(ctx).withSkipVerification(true);
+
+ DUUIUIMADriver uima_driver = new DUUIUIMADriver();
+
+// A driver must be added before components can be added for it in the composer. After that the composer is able to use the individual drivers.
+ composer.addDriver(uima_driver);
+// Adding a UIMA annotator for writing the result of the pipeline as XMI files.
+ composer.add(new DUUIUIMADriver.Component(
+ createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/test"
+ )).withScale(iWorkers));
+
+// The document is processed through the pipeline. In addition, files of entire repositories can be processed.
+ composer.run(jc);
+ }
+
+
+ @Test
+ public void universTest2() throws Exception {
+
+ int iWorkers = 5; // define the number of workers
+
+ JCas jc = JCasFactory.createText("hello world");
+ DocumentMetaData dmd = new DocumentMetaData(jc);
+ dmd.setDocumentId("test01");
+ dmd.setDocumentTitle("univers test2");
+ dmd.addToIndexes();
+
+
+ //
+
+
+ DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary();
+ DUUILuaSandbox sandbox = new DUUILuaSandbox();
+ sandbox._allowAllJavaClasses = true;
+ ctx.withSandbox(sandbox);
+ DUUIComposer composer = new DUUIComposer()
+ .withSkipVerification(true)
+ .withLuaContext(ctx)
+ .withWorkers(iWorkers);
+
+ DUUIUIMADriver uimaDriver = new DUUIUIMADriver();
+ DUUIDockerDriver duuidDriver = new DUUIDockerDriver();
+ composer.addDriver(uimaDriver, duuidDriver);
+ // reset also init make sure before run empty
+ composer.resetPipeline();
+
+
+ composer.add(new DUUIDockerDriver.Component("docker.texttechnologylab.org/textimager-duui-spacy-single-de_core_news_sm:0.1.4")
+ .withImageFetching()
+ .withScale(iWorkers)
+ .build());
+
+ composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class,
+ XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/",
+ XmiWriter.PARAM_PRETTY_PRINT, true,
+ XmiWriter.PARAM_OVERWRITE, true,
+ XmiWriter.PARAM_VERSION, "1.1",
+ XmiWriter.PARAM_COMPRESSION, "GZIP"))
+ .build());
+
+ composer.run(jc);
+ }
+
+
+}