From ef0ef7ae93dc8fbf7739c0aa555938340810d736 Mon Sep 17 00:00:00 2001 From: Jiadong Zhou Date: Mon, 9 Jun 2025 20:18:45 +0200 Subject: [PATCH 1/5] move codes in a new repo --- pom.xml | 30 +- src/.idea/misc.xml | 6 + src/.idea/modules.xml | 10 + src/.idea/vcs.xml | 6 + src/.idea/workspace.xml | 54 ++ .../driver/DUUIPipelineComponent.java | 253 ++++++ .../driver/slurm/DUUISlurmDriver.java | 816 ++++++++++++++++++ .../driver/slurm/DUUISlurmInterface.java | 106 +++ .../driver/slurm/Portmanager.java | 73 ++ .../driver/slurm/SinfoData.java | 149 ++++ .../driver/slurm/SlurmConfig.java | 84 ++ .../driver/slurm/SlurmProcess.java | 274 ++++++ .../driver/slurm/SlurmUtils.java | 272 ++++++ src/test/java/SlurmTests.java | 68 ++ 14 files changed, 2194 insertions(+), 7 deletions(-) mode change 100644 => 100755 pom.xml create mode 100644 src/.idea/misc.xml create mode 100644 src/.idea/modules.xml create mode 100644 src/.idea/vcs.xml create mode 100644 src/.idea/workspace.xml create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java create mode 100644 src/test/java/SlurmTests.java diff --git a/pom.xml b/pom.xml old mode 100644 new mode 100755 index ff3429e9..65274e3b --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.texttechnologylab DockerUnifiedUIMAInterface - 1.5.0 + 1.4.6 Automatic analysis of large text corpora is a complex task. This complexity particularly concerns the question of time efficiency. Furthermore, efficient, flexible, and extensible textanalysis requires the continuous integration of every new text analysis tools. Since there are currently, in the area of NLP and especially in the application context of UIMA, only very few to no adequate frameworks for these purposes, which are not simultaneously outdated or can no longer be used for security reasons, this work will present a new approach to fill this gap. To this end, we present Docker Unified UIMA Interface (DUUI), a scalable, flexible, lightweight, and featurerich framework for automated and distributed analysis of text corpora that leverages experience in Big Data analytics and virtualization with Docker. @@ -77,7 +77,7 @@ UTF-8 21 21 - 3.0.8 + 3.0.7 3.0.0 3.5.0 2.4.0 @@ -260,11 +260,11 @@ - - - - - + + org.apache.httpcomponents.client5 + httpclient5 + 5.3.1 + @@ -641,6 +641,22 @@ jsoup 1.17.2 + + com.github.docker-java + docker-java-transport-httpclient5 + 3.5.0 + + + org.assertj + assertj-core + 3.24.2 + + + org.testng + testng + RELEASE + test + diff --git a/src/.idea/misc.xml b/src/.idea/misc.xml new file mode 100644 index 00000000..47478b91 --- /dev/null +++ b/src/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/.idea/modules.xml b/src/.idea/modules.xml new file mode 100644 index 00000000..31eaf94d --- /dev/null +++ b/src/.idea/modules.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/src/.idea/vcs.xml b/src/.idea/vcs.xml new file mode 100644 index 00000000..6c0b8635 --- /dev/null +++ b/src/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/.idea/workspace.xml b/src/.idea/workspace.xml new file mode 100644 index 00000000..f1bdb6ee --- /dev/null +++ b/src/.idea/workspace.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + 1749492335426 + + + + + + \ No newline at end of file diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java index 4329ac9a..e9eb7509 100644 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java @@ -73,6 +73,21 @@ public class DUUIPipelineComponent { private static String targetView = "targetView"; private static String timeout = "timeout"; + private static String slurmJobName = "slurmJobName"; + private static String slurmImagePort = "slurmPort"; + //private static String slurmHostPort = "slurmHostPort"; + private static String slurmRuntime = "slurmRuntime"; + private static String slurmCpus = "slurmCpus"; + private static String slurmMemory = "slurmMemory"; + private static String slurmErrorLocation = "slurmErrorLocation"; + private static String slurmOutPutLocation = "slurmOutPutLocation"; + private static String slurmSIFDiskLocation = "slurmSIFDiskLocation"; + private static String slurmSIFRepoLocation = "slurmSIFRepoLocation"; + private static String slurmGPU = "slurmGPU"; + private static String slurmSIFImageName = "slurmSIFImageName"; + private static String slurmEntryLocation = "slurmEntryLocation"; + private static String slurmNoShutdown = "slurmNoShutdown"; + private static String slurmUvicorn = "slurmUvicorn"; private String getVersion() throws URISyntaxException, IOException { ClassLoader classLoader = DUUIPipelineComponent.class.getClassLoader(); try { @@ -667,4 +682,242 @@ public DUUISegmentationStrategy getSegmentationStrategy() { public long getTimeout() { return Long.valueOf(_parameters.getOrDefault(timeout, "60")); } + + //1 + public DUUIPipelineComponent withSlurmJobName(String jobName) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (jobName == null) { + _options.remove(slurmJobName); + return this; + } + _options.put(slurmJobName, jobName); + _options.put(componentName, jobName); + return this; + } + //2 + public DUUIPipelineComponent withSlurmImagePort(String port) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (port == null) { + _options.remove(slurmImagePort); + return this; + } + _options.put(slurmImagePort, port); + return this; + } +//3 + + // public DUUIPipelineComponent withSlurmHostPort(String port) { +// if (_finalizedEncoded != null) { +// throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); +// } +// if (port == null) { +// _options.remove(slurmHostPort); +// return this; +// } +// _options.put(slurmHostPort, port); +// return this; +// } +//4 + public DUUIPipelineComponent withSlurmRuntime(String time) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (time == null) { + _options.remove(slurmRuntime); + return this; + } + _options.put(slurmRuntime, time); + return this; + } + + //5 + public DUUIPipelineComponent withSlurmCpus(String num) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (num == null) { + _options.remove(slurmCpus); + return this; + } + _options.put(slurmCpus, num); + return this; + } + + //6 + public DUUIPipelineComponent withSlurmMemory(String numG) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (numG == null) { + _options.remove(slurmMemory); + return this; + } + _options.put(slurmMemory, numG); + return this; + } + //7 + public DUUIPipelineComponent withSlurmOutPutLocation(String loc) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (loc == null) { + _options.remove(slurmOutPutLocation); + return this; + } + _options.put(slurmOutPutLocation, loc); + return this; + } + //8 + public DUUIPipelineComponent withSlurmErrorLocation(String loc) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (loc == null) { + _options.remove(slurmErrorLocation); + return this; + } + _options.put(slurmErrorLocation, loc); + return this; + } + //9 + public DUUIPipelineComponent withSlurmSaveIn(String saveTo) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (saveTo == null) { + _options.remove(slurmSIFDiskLocation); + return this; + } + _options.put(slurmSIFDiskLocation, saveTo); + return this; + } + + //10 + public DUUIPipelineComponent withSlurmGPU(String num) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (num == null) { + _options.remove(slurmGPU); + return this; + } + _options.put(slurmGPU, num); + return this; + } + //11 + public DUUIPipelineComponent withSlurmSIFName(String sifName) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (sifName == null) { + _options.remove(slurmSIFImageName); + return this; + } + _options.put(slurmSIFImageName, sifName); + return this; + + } + //12 + public DUUIPipelineComponent withSlurmEntryLocation(String loc) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (loc == null) { + _options.remove(slurmEntryLocation); + return this; + } + _options.put(slurmEntryLocation, loc); + return this; + } + public DUUIPipelineComponent withSlurmUvicorn(String loc) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (loc == null) { + _options.remove(slurmUvicorn); + return this; + } + _options.put(slurmUvicorn, loc); + return this; + } + + public DUUIPipelineComponent withSlurmRepoLocation(String loc) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (loc == null) { + _options.remove(slurmSIFRepoLocation); + return this; + } + _options.put(slurmSIFRepoLocation, loc); + return this; + + } + + + public String getSlurmSIFImageName() { + return _options.get(slurmSIFImageName); + } + + public String getSlurmGPU() { + return _options.get(slurmGPU); + } + + public String getSlurmSIFDiskLocation() { + return _options.get(slurmSIFDiskLocation); + } + + public String getSlurmOutPutLocation() { + return _options.get(slurmOutPutLocation); + } + + public String getSlurmMem() { + return _options.get(slurmMemory); + } + + public String getSlurmCpus() { + return _options.get(slurmCpus); + } + + public String getSlurmRuntime() { + return _options.get(slurmRuntime); + } + + public String getSlurmImagePort() { + return _options.get(slurmImagePort); + } + // public String getSlurmHostPort() { +// return _options.get(slurmHostPort); +// } + public String getSlurmJobName() { + return _options.get(slurmJobName); + } + public String getSlurmErrorLocation() { + return _options.get(slurmErrorLocation); + } + public String getSlurmEntryLocation() { + return _options.get(slurmEntryLocation); + + } + + public Boolean getSlurmRunAfterExit(Boolean defaultValue) { + String result = _options.get(slurmNoShutdown); + if (result == null) return defaultValue; + return Boolean.parseBoolean(result); + } + + public String getSlurmSIFRepoLocation() { + return _options.get(slurmSIFRepoLocation); + + } + public String getSlurmUvicorn() { + return _options.get(slurmUvicorn); + + } + + } diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java new file mode 100755 index 00000000..76677d95 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java @@ -0,0 +1,816 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.uima.UIMAException; +import org.apache.uima.cas.CASException; +import org.apache.uima.fit.factory.JCasFactory; +import org.apache.uima.jcas.JCas; +import org.apache.uima.resource.ResourceInitializationException; +import org.apache.uima.resource.metadata.TypeSystemDescription; +import org.apache.uima.util.TypeSystemUtil; +import org.javatuples.Triplet; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIDockerInterface; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIFallbackCommunicationLayer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.IDUUICommunicationLayer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.DUUIWebsocketAlt; +import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.IDUUIConnectionHandler; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.*; +import org.texttechnologylab.DockerUnifiedUIMAInterface.exception.PipelineComponentException; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaCommunicationLayer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +import org.texttechnologylab.DockerUnifiedUIMAInterface.pipeline_storage.DUUIPipelineDocumentPerformance; +import org.texttechnologylab.DockerUnifiedUIMAInterface.segmentation.DUUISegmentationStrategy; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.InvalidParameterException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +import static java.lang.String.format; +import static org.bouncycastle.oer.its.ieee1609dot2.SignerIdentifier.digest; +import static org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.PortManager.waitUntilReachable; + +public class DUUISlurmDriver implements IDUUIDriverInterface { + // Man kann den Docker-Client dazu verwenden, ein Image herunterzuladen (per Pull). + private DUUIDockerInterface _docker_interface; + private DUUILuaContext _luaContext; + private DUUISlurmInterface _interface; + + private int _http_timeout_s = 10; + private HashMap _active_components; + private HttpClient _client; + private IDUUIConnectionHandler _wsclient; + private final static Logger LOGGER = Logger.getLogger(DUUIComposer.class.getName()); + + //A + public DUUISlurmDriver() throws IOException, UIMAException, SAXException { + _interface = new DUUISlurmInterface(); + _client = HttpClient.newHttpClient(); + + JCas _basic = JCasFactory.createJCas(); + _basic.setDocumentLanguage("en"); + _basic.setDocumentText("Hello World!"); + //_http_timeout = 10000; + TypeSystemDescription desc = TypeSystemUtil.typeSystem2TypeSystemDescription(_basic.getTypeSystem()); + StringWriter wr = new StringWriter(); + // trigger for type checking + desc.toXML(wr); + _active_components = new HashMap(); + _luaContext = null; + } + + //B + + /** + * Creation of the communication layer based on the Driver + * + * @param url localhost + * @param jc jcas + * @param http_timeout_s timeout with fastapi + * @param client communication wtih fastapi + * @param context lua + * @param skipVerification true + * @return + * @throws Exception + */ + public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, int http_timeout_s, HttpClient client, DUUILuaContext context, boolean skipVerification) throws Exception { + long start = System.currentTimeMillis(); + // default http + IDUUICommunicationLayer layer = new DUUIFallbackCommunicationLayer(); // Hier wird layer zum ersten mal erstellt. + boolean fatal_error = false; + + // error counter + int iError = 0; + while (true) { + HttpRequest request = null; + try {// req + // get lua script + request = HttpRequest.newBuilder() + .uri(URI.create(url + DUUIComposer.V1_COMPONENT_ENDPOINT_COMMUNICATION_LAYER)) + .version(HttpClient.Version.HTTP_1_1) +// .timeout(Duration.ofSeconds(10)) + .timeout(Duration.ofSeconds(http_timeout_s)) + .GET() + .build(); + } catch (Exception e) { + e.printStackTrace(); + } + try { + HttpResponse resp = null; + + boolean connectionError = true; + int iCount = 0; + //max.10 + while (connectionError && iCount < 10) { + + try { + + // req lua + resp = client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join(); + + connectionError = false; + } catch (Exception e) { + System.out.println(e.getMessage() + "\t" + url); + if (e instanceof java.net.ConnectException) { + Thread.sleep(http_timeout_s); + iCount++; + } else if (e instanceof CompletionException) { + Thread.sleep(http_timeout_s); + iCount++; + } + } + } + // if success? + if (resp.statusCode() == 200) { + // parse script as string + String body2 = new String(resp.body(), Charset.defaultCharset()); + try { + //printfunc.operation("Component lua communication layer, loading..."); + // new lua commu layer + IDUUICommunicationLayer lua_com = new DUUILuaCommunicationLayer(body2, "requester", context); + // replace + layer = lua_com; + + //printfunc.operation("Component lua communication layer, loaded."); + break; + } catch (Exception e) { + fatal_error = true; + e.printStackTrace(); + throw new Exception("Component provided a lua script which is not runnable."); + } + // fallback + } else if (resp.statusCode() == 404) { + // printfunc.operation("Component provided no own communication layer implementation using fallback."); + break; + } + long finish = System.currentTimeMillis(); + long timeElapsed = finish - start; + if (timeElapsed > http_timeout_s) { + throw new TimeoutException(format("The Container did not provide one succesful answer in %d milliseconds", http_timeout_s)); + } + + } catch (Exception e) { + + if (fatal_error) { + throw e; + } else { + Thread.sleep(2000l); + iError++; + } + + if (iError > 10) { + throw e; + } + } + } + if (skipVerification) { + return layer; + } + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + // cas seri + try { + //TODO: Make this accept options to better check the instantiation! + layer.serialize(jc, stream, null, "_InitialView"); + } catch (Exception e) { + e.printStackTrace(); + throw new Exception(format("The serialization step of the communication layer fails for implementing class %s", layer.getClass().getCanonicalName())); + } +// http req + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS)) + .version(HttpClient.Version.HTTP_1_1) + .POST(HttpRequest.BodyPublishers.ofByteArray(stream.toByteArray())) + .build(); + // resp + HttpResponse resp = client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join(); + + if (resp.statusCode() == 200) { + ByteArrayInputStream inputStream = new ByteArrayInputStream(resp.body()); + try { + layer.deserialize(jc, inputStream, "_InitialView"); + } catch (Exception e) { + System.err.printf("Caught exception printing response %s\n", new String(resp.body(), StandardCharsets.UTF_8)); + throw e; + } + // use for shallow copy + return layer; + } else { + throw new Exception(format("The container returned response with code != 200\nResponse %s", resp.body().toString())); + } + } + + + //C + public void setLuaContext(DUUILuaContext luaContext) { + _luaContext = luaContext; + } + + //------------------------------------------------------------------------------------------------------------- + //D no use +//------------------------------------------------------------------------------------------------------------- + //E + public boolean canAccept(DUUIPipelineComponent comp) { + return comp.getSlurmSIFImageName() != null; + } +//------------------------------------------------------------------------------------------------------------- + //F + + /** + * Instantiate the component + * + * @param component + * @param jc + * @param skipVerification + * @return + * @throws Exception + */ + public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skipVerification, AtomicBoolean shutdown) throws InterruptedException, URISyntaxException, IOException { + String uuid = UUID.randomUUID().toString(); + while (_active_components.containsKey(uuid.toString())) { + uuid = UUID.randomUUID().toString(); + } + + + DUUISlurmDriver.InstantiatedComponent comp = new DUUISlurmDriver.InstantiatedComponent(component); + + // Inverted if check because images will never be pulled if !comp.getImageFetching() is checked. + + + +// if (!(component.getSlurmSIFRepoLocation() == null)) { +// // mean local no sif file +// //_docker_interface.pullImage(component.getDockerImageName()); +// // 自动把sif iamge name 传入 dockerimagename +// SlurmUtils.pullSifImagefromRemoteDockerRepo(component.getSlurmSIFImageName(), component.getDockerImageName()); +// } +// // +// else + + // etwas neues + // + if (!(component.getSlurmSIFDiskLocation() == null)) { + //SlurmUtils.pullSifImagefromLocalDockerRepo(component.getSlurmSIFImageName(),component.getSlurmSIFImageName()); + + + System.out.println("sif file existed"); + } else { + throw new RuntimeException("SIF File conflict detected, either local disk location or remote repo url "); + } + System.out.printf("[SlurmDriver] Assigned new pipeline component unique id %s\n", uuid); +// String digest = _interface.getDigestFromImage(comp.getImageName()); + + //comp.getPipelineComponent().__internalPinDockerImage(comp.getImageName(), digest); + //System.out.printf("[DockerLocalDriver] Transformed image %s to pinnable image name %s\n", comp.getImageName(), comp.getPipelineComponent().getDockerImageName()); + + _active_components.put(uuid, comp); + + for (int i = 0; i < comp.getScale(); i++) { + if (shutdown.get()) { + return null; + } +// + int hostPort = PortManager.acquire(); + System.out.println("prepare to reserve port " + hostPort); + // dient als placeholder + try (PortManager.PortReservation r = new PortManager.PortReservation(hostPort)) { + // + // + // placeholder muss vor dem Lauf eines Komponent freilassen werden, sonst entsteht ein Konflikt + r.close(); + String jobID = _interface.run(component, hostPort); + + String port = _interface.extractPort(jobID); // + if (!port.equals(Integer.toString(hostPort))) { + throw new RuntimeException("port mismatch"); + } + + + + //wait test if ok pass + waitUntilReachable("127.0.0.1", hostPort, Duration.ofMinutes(1)); + + try { + final int iCopy = i; + final String uuidCopy = uuid; + IDUUICommunicationLayer layer = responsiveAfterTime("http://127.0.0.1" + ":" + port, jc, _http_timeout_s, _client, _luaContext, skipVerification); + + if (comp.isWebsocket()) { + String url = "ws://127.0.0.1:" + port; + _wsclient = new DUUIWebsocketAlt( + url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS_WEBSOCKET, comp.getWebsocketElements()); + } else { + _wsclient = null; + } + /** + * @see + * @edited + * Dawit Terefe + * + * Saves websocket client in ComponentInstance for + * retrieval in process_handler-function. + */ + comp.addInstance(new DUUISlurmDriver.ComponentInstance(jobID, hostPort, layer, _wsclient)); + } catch (Exception e) { + //_interface.stop_container(containerid); + //throw e; + } + } + } + return shutdown.get() ? null : uuid; + } + + @Override + public void printConcurrencyGraph(String uuid) { + + } + + +////G no use +// /** +// * Show the maximum parallelism +// * +// * @param uuid +// */ +// public void printConcurrencyGraph(String uuid) { +// DUUISlurmDriver.InstantiatedComponent component = _active_components.get(uuid); +// if (component == null) { +// throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); +// } +// System.out.printf("[DockerLocalDriver][%s]: Maximum concurrency %d\n", uuid, component.getInstances().size()); +// } + +//----------------------------------------------------------------------------------------------------------------------- +//H uuid 1--------------------n same type component + + /** + * Return the TypeSystem used by the given Component + * + * @param uuid + * @return + * @throws InterruptedException + * @throws IOException + * @throws SAXException + * @throws CompressorException + * @throws ResourceInitializationException + */ + public TypeSystemDescription get_typesystem(String uuid) throws InterruptedException, IOException, SAXException, CompressorException, ResourceInitializationException { + DUUISlurmDriver.InstantiatedComponent comp = _active_components.get(uuid); + if (comp == null) { + throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); + } + return IDUUIInstantiatedPipelineComponent.getTypesystem(uuid, comp); + } + + @Override + public int initReaderComponent(String uuid, Path filePath) throws Exception { + return 0; + } + + + /// /I what is this? +// /** +// * init reader component +// * @param uuid +// * @param filePath +// * @return +// */ +// @Override +// public int initReaderComponent(String uuid, Path filePath) { +// InstantiatedComponent comp = _active_components.get(uuid); +// if (comp == null) { +// throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); +// } +// return IDUUIInstantiatedPipelineReaderComponent.initComponent(comp, filePath); +// } +//------------------------------------------------------------------------------------------------------------- +//J over write service logic + + + //K no use + +//L close no use + + //M + public static class ComponentInstance implements IDUUIUrlAccessible { + private String _job_ID; + private int _hostPort; + private IDUUIConnectionHandler _handler; + private IDUUICommunicationLayer _communicationLayer; + + public ComponentInstance(String id, int hostPort, IDUUICommunicationLayer communicationLayer) { + _job_ID = id; + _hostPort = hostPort; + _communicationLayer = communicationLayer; + } + + public ComponentInstance(String id, int hostPort, IDUUICommunicationLayer communicationLayer, IDUUIConnectionHandler handler) { + _job_ID = id; + _hostPort = hostPort; + + _communicationLayer = communicationLayer; + _handler = handler; + } + + public IDUUICommunicationLayer getCommunicationLayer() { + return _communicationLayer; + } + + + public String get_job_ID() { + return _job_ID; + } + + public int get_hostPort() { + return _hostPort; + } + +// public int get_fastAPIPort() { +// return _fastAPIPort; +// } + + public IDUUIConnectionHandler get_handler() { + return _handler; + } + + public IDUUICommunicationLayer get_communicationLayer() { + return _communicationLayer; + } + + public String generateURL() { + return format("http://127.0.0.1" + ":%d", _hostPort); + } + + public String generateURL(String localhost) { + return format("http://localhost" + ":%d", _hostPort); + } + + public IDUUIConnectionHandler getHandler() { + return _handler; + } + } + + //------------------------------------------------------------------------------------ +//J + + /** + * Execute a component in the driver + * + * @param uuid + * @param aCas + * @param perf + * @param composer + * @throws CASException + */ + public void run(String uuid, JCas aCas, DUUIPipelineDocumentPerformance perf, DUUIComposer composer) throws CASException, PipelineComponentException { + long mutexStart = System.nanoTime(); + DUUISlurmDriver.InstantiatedComponent comp = _active_components.get(uuid); + if (comp == null) { + throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); + } + if (comp.isWebsocket()) { + IDUUIInstantiatedPipelineComponent.process_handler(aCas, comp, perf); + } else { + IDUUIInstantiatedPipelineComponent.process(aCas, comp, perf); + } + } + + @Override + public boolean destroy(String uuid) { + return false; + } + + //N + static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent { + private String _image_name; + private ConcurrentLinkedQueue _instances; + private boolean _gpu; + private boolean _keep_runnging_after_exit; + private int _scale; + private boolean _withImageFetching; + private boolean _websocket; + private int _ws_elements; + + //private String _reg_password; + //private String _reg_username; + private String _uniqueComponentKey; + private Map _parameters; + private String _sourceView; + private String _targetView; + private DUUIPipelineComponent _component; + + // 这里？？ + public Triplet getComponent() { + long mutexStart = System.nanoTime(); + DUUISlurmDriver.ComponentInstance inst = _instances.poll(); + while (inst == null) { + inst = _instances.poll(); + } + long mutexEnd = System.nanoTime(); + return Triplet.with(inst, mutexStart, mutexEnd); + } + + //-------------------------------------------------------------------------------- + public void addComponent(IDUUIUrlAccessible access) { + _instances.add((DUUISlurmDriver.ComponentInstance) access); + } + + InstantiatedComponent(DUUIPipelineComponent comp) { + _component = comp; + _parameters = comp.getParameters(); + _targetView = comp.getTargetView(); + _sourceView = comp.getSourceView(); + + _withImageFetching = comp.getDockerImageFetching(false); + + _uniqueComponentKey = ""; + + + _instances = new ConcurrentLinkedQueue(); + + _scale = comp.getScale(1); + + _gpu = comp.getDockerGPU(false); + + _keep_runnging_after_exit = comp.getSlurmRunAfterExit(false); + + // _reg_password = comp.getDockerAuthPassword(); + // _reg_username = comp.getDockerAuthUsername(); + + _websocket = comp.isWebsocket(); + _ws_elements = comp.getWebsocketElements(); + } + + public DUUIPipelineComponent getPipelineComponent() { + return _component; + } + + public String getUniqueComponentKey() { + return _uniqueComponentKey; + } + +// public String getPassword() { +// return _reg_password; +// } + + // public String getUsername() { + // return _reg_username; + // } + + public boolean getImageFetching() { + return _withImageFetching; + } + + public String getImageName() { + return _image_name; + } + + public int getScale() { + return _scale; + } + + public boolean getRunningAfterExit() { + return _keep_runnging_after_exit; + } + + public void addInstance(DUUISlurmDriver.ComponentInstance inst) { + _instances.add(inst); + } + + public boolean usesGPU() { + return (Integer.parseInt(_component.getSlurmGPU()) > 0) ? true : false; + } + + public ConcurrentLinkedQueue getInstances() { + return _instances; + } + + public Map getParameters() { + return _parameters; + } + + public String getSourceView() { + return _sourceView; + } + + public String getTargetView() { + return _targetView; + } + + public boolean isWebsocket() { + return _websocket; + } + + public int getWebsocketElements() { + return _ws_elements; + } + } + + + //O + public static class Component { + private DUUIPipelineComponent _component; + + public DUUISlurmDriver.Component withParameter(String key, String value) { + _component.withParameter(key, value); + return this; + } + + public DUUISlurmDriver.Component withView(String viewName) { + _component.withView(viewName); + return this; + } + + public DUUISlurmDriver.Component withSourceView(String viewName) { + _component.withSourceView(viewName); + return this; + } + + public DUUISlurmDriver.Component withTargetView(String viewName) { + _component.withTargetView(viewName); + return this; + } + + public Component(String target) throws URISyntaxException, IOException { + _component = new DUUIPipelineComponent(); + _component.withDockerImageName(target); + } + + public Component(DUUIPipelineComponent pComponent) throws URISyntaxException, IOException { + _component = pComponent; + _component.withDriver(DUUISlurmDriver.class); + } + + public DUUISlurmDriver.Component withDescription(String description) { + _component.withDescription(description); + return this; + } + + public DUUISlurmDriver.Component withScale(int scale) { + _component.withScale(scale); + return this; + } + +// public DUUISlurmDriver.Component withRegistryAuth(String username, String password) { +// _component.withDockerAuth(username, password); +// return this; +// } + +// public DUUISlurmDriver.Component withImageFetching() { +// return withImageFetching(true); +// } +// +// public DUUISlurmDriver.Component withImageFetching(boolean imageFetching) { +// _component.withDockerImageFetching(imageFetching); +// return this; +// } + +// public DUUISlurmDriver.Component withGPU(boolean gpu) { +// _component.withDockerGPU(gpu); +// return this; +// } + +// public DUUISlurmDriver.Component withRunningAfterDestroy(boolean run) { +// _component.withDockerRunAfterExit(run); +// return this; +// } + +// public DUUISlurmDriver.Component withWebsocket(boolean b) { +// _component.withWebsocket(b); +// return this; +// } +// +// public DUUISlurmDriver.Component withWebsocket(boolean b, int elements) { +// _component.withWebsocket(b, elements); +// return this; +// } + + public DUUISlurmDriver.Component withSegmentationStrategy(DUUISegmentationStrategy strategy) { + _component.withSegmentationStrategy(strategy); + return this; + } + + public DUUISlurmDriver.Component withSegmentationStrategy(Class strategyClass) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + _component.withSegmentationStrategy(strategyClass.getDeclaredConstructor().newInstance()); + return this; + } + + public DUUIPipelineComponent build() { + return _component; + } + +// public DUUISlurmDriver.Component withName(String name) { +// _component.withName(name); +// return this; +// } + + + //1 + public Component withSlurmJobName(String jobName) { + _component.withSlurmJobName(jobName); + return this; + } + + //2 + public Component withSlurmImagePort(String port) { + _component.withSlurmImagePort(port); + return this; + } + + + //3 +// public Component withSlurmHostPort(String port) { +// _component.withSlurmHostPort(port); +// return this; +// } + + //4 + public Component withSlurmRuntime(String time) { + _component.withSlurmRuntime(time); + return this; + } + + //5 + public Component withSlurmCpus(String num) { + _component.withSlurmCpus(num); + return this; + } + + //6 + public Component withSlurmMemory(String numG) { + _component.withSlurmMemory(numG); + return this; + } + + //7 + public Component withSlurmOutPutLocation(String loc) { + _component.withSlurmOutPutLocation(loc); + return this; + } + + //8 + public Component withSlurmErrorLocation(String loc) { + _component.withSlurmErrorLocation(loc); + return this; + } + + //9 + public Component withSlurmSaveIn(String saveTo) { + _component.withSlurmSaveIn(saveTo); + return this; + } + + //10 + public Component withSlurmGPU(String num) { + _component.withSlurmGPU(num); + return this; + } + + //11 + public Component withSlurmSIFName(String sifName) { + _component.withSlurmSIFName(sifName); + return this; + + } + + //12 + public Component withSlurmEntryLocation(String loc) { + _component.withSlurmEntryLocation(loc); + return this; + + } + + + } + +//-------------------------------------------------------------------------- + + + @Override + public void shutdown() { + + } +} + + diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java new file mode 100755 index 00000000..12ae6701 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java @@ -0,0 +1,106 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.core.DockerClientBuilder; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; + +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.HashMap; + +// funtionen: 1 use docker cli pull image falls nicht vorhanden +// 2 in slurm hat jede Aufgabe eindeutig ein jobid, man kann anhand id ein job starten oder stoppen， +// also eine Verwalttung von Port und id sehr wichtig Hashmap +public class DUUISlurmInterface { + + private DockerClient _docker; + + private HashMap _jobID_PortMap = new HashMap<>(); + + public DUUISlurmInterface() throws IOException { +// + if (!System.getProperty("os.name").contains("Windows")) { + _docker = DockerClientBuilder.getInstance().build(); + } else { + // Windows + DockerHttpClient http = null; + try { + http = new ApacheDockerHttpClient.Builder() + .connectionTimeout(Duration.ofSeconds(5)) + .responseTimeout(Duration.ofMinutes(10)) + .dockerHost(URI.create("npipe:////./pipe/docker_engine")) + .build(); + } catch (Exception e) { + http = new ApacheDockerHttpClient.Builder() + .connectionTimeout(Duration.ofSeconds(5)) + .responseTimeout(Duration.ofMinutes(10)) + .dockerHost(URI.create("tcp://127.0.0.1:2375")) // if npipe doesn't work. + .build(); + } + _docker = DockerClientBuilder.getInstance() + .withDockerHttpClient(http) + .build(); + } + + + } + + + public boolean checkDependencies() { + return SlurmUtils.checkSlurmInstalled() && SlurmUtils.checkSocatInstalled(); + } + + // Die Funktion run erstellt ein Shell-Skript, das dem Slurm-Jobsystem übergeben und dann ausgeführt wird.“ + public String run(DUUIPipelineComponent comp, int hostPort) throws IOException, InterruptedException { + String gpu = comp.getSlurmGPU(); + String sifName = comp.getSlurmSIFImageName(); + String slurmJobName = comp.getSlurmJobName(); + String slurmEntryLocation = comp.getSlurmEntryLocation(); + String slurmErrorLocation = comp.getSlurmErrorLocation(); + String slurmImagePort = comp.getSlurmImagePort(); + String slurmCpus = comp.getSlurmCpus(); + String slurmRuntime = comp.getSlurmRuntime(); + String slurmMem = comp.getSlurmMem(); + String slurmSaveIn = comp.getSlurmSIFDiskLocation(); + String slurmOutPutLocation = comp.getSlurmOutPutLocation(); + String slurmUvicorn = comp.getSlurmUvicorn(); +//Diese Funktion konvertiert einen String in ein Shell Format. + String jobid = SlurmUtils.submitJob( + slurmJobName, + Integer.toString(hostPort), + slurmCpus, gpu, slurmEntryLocation, + slurmErrorLocation, + slurmImagePort, + slurmMem, + slurmRuntime, + slurmOutPutLocation, + sifName, + slurmSaveIn, + slurmUvicorn + ); + + System.out.println("job submitted ready for release, jobid : " + jobid); + // neue Abbildung gespeichert + _jobID_PortMap.put(jobid, Integer.toString(hostPort)); + + return jobid; + } + + + public String extractPort(String jobid){ + return _jobID_PortMap.get(jobid); + } + + public DockerClient get_docker() { + return _docker; + } + + public HashMap get_jobID_PortMap() { + return _jobID_PortMap; + } + +} + diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java new file mode 100755 index 00000000..52ff9c1a --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java @@ -0,0 +1,73 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.time.Duration; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/* 20000–30000 speiziell reserviert fuer slurm */ +final class PortManager { + private static final int BASE = 20000; + private static final int MAX = 30000; + private static final Set RESERVED = ConcurrentHashMap.newKeySet(); + + /** get a free port for placeholder -> make it RESERVED */ + + static int acquire() throws IOException { + for (int p = BASE; p < MAX; p++) { + if (!RESERVED.add(p)) continue; // conflict + if (isFree(p)) return p; // success + RESERVED.remove(p); // bind failed + } + throw new IOException("No free port in pool"); + } + + static void release(int port) { RESERVED.remove(port); } + + private static boolean isFree(int port) { + try (ServerSocket s = new ServerSocket()) { + s.setReuseAddress(false); + s.bind(new InetSocketAddress("0.0.0.0", port)); + return true; + } catch (IOException e) { return false; } + } + + // ich vergesse immer am ende close()anzurufen, benuztze autocloseable, anweisungen in {} verklammern + static class PortReservation implements AutoCloseable { + private final ServerSocket placeholder; + final int port; + + PortReservation(int port) throws IOException { + this.port = port; + this.placeholder = new ServerSocket(); + this.placeholder.setReuseAddress(false); + this.placeholder.bind(new InetSocketAddress("0.0.0.0", port)); + } + @Override public void close() throws IOException { placeholder.close(); } + } +// schickt ein paar + + static void waitUntilReachable(String host, int port, Duration timeout) + throws IOException, InterruptedException { + + long deadline = System.nanoTime() + timeout.toNanos(); + long backoff = 250; // ms!!! + while (System.nanoTime() < deadline) { + try (Socket s = new Socket()) { + s.connect(new InetSocketAddress(host, port), 1000); + return; + } catch (IOException ignore) { } + Thread.sleep(backoff); + backoff = Math.min(backoff * 2, 2000); + } + throw new IOException("Port "+port+" did not open within "+timeout.toSeconds()+" s"); + } + + + + + +} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java new file mode 100755 index 00000000..eecae4d2 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java @@ -0,0 +1,149 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.List; + +/** + * cli insruction sinfo in java console + * für zukunftige DEBUG und log + */ +public class SinfoData { + + public static class SinfoResponse { + public List sinfo = List.of(); + public Meta meta = new Meta(); + public List errors = List.of(); + public List warnings = List.of(); + } + + public static class SinfoRecord { + public int port; + public Nodes nodes = new Nodes(); + public Cpus cpus = new Cpus(); + public Node node = new Node(); + public Gres gres = new Gres(); + public Memory memory = new Memory(); + public Partition partition = new Partition(); + + public String comment = "", cluster = "", extra = "", reservation = ""; + } + + + public static class Node { + public List state = List.of(); + } + + public static class Nodes { + public int total, idle, allocated, other; + public List hostnames = List.of(), addresses = List.of(), nodes = List.of(); + } + + public static class Cpus { + public int total, idle, allocated, other; + } + + public static class Gres { + public String total = "", used = ""; + } + + public static class Memory { + public int minimum, maximum, allocated; + public Free free = new Free(); + + public static class Free { + public NumberObj minimum = new NumberObj(), maximum = new NumberObj(); + + public static class NumberObj { + public boolean set; + public boolean infinite; + public int number; + } + } + } + + public static class Partition { + public String name = ""; + public PartitionState partition = new PartitionState(); + + public static class PartitionState { + public List state = List.of(); + } + } + + public static class Meta { + public Slurm slurm = new Slurm(); + + public static class Slurm { + public String release = ""; + } + } + + + private static SinfoResponse runSinfo() { + StringBuilder json = new StringBuilder(); + try { + Process p = new ProcessBuilder("sinfo", "--json").redirectErrorStream(true).start(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + br.lines().forEach(json::append); + } + p.waitFor(); + } catch (Exception e) { + throw new RuntimeException("sinfo failed", e); + } + + ObjectMapper mapper = new ObjectMapper() + + .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)) + + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + try { + return mapper.readValue(json.toString(), SinfoResponse.class); + } catch (Exception e) { + throw new RuntimeException("parse json fail...", e); + } + } + + public static void sinfo() { + SinfoResponse resp = runSinfo(); + + String hdrFmt = "%-10s %10s %6s %10s %10s %-10s %-10s %11s %9s %11s%n"; + String rowFmt = "%-10s %6s %8s %10s %10s %-12s %-10s %9d %8d %10d%n"; + + System.out.printf(hdrFmt, + "Partition", "NodeTotal", "Idle", "CPU_Total", "CPU_Idle", + "State", "GRES", "MemTot(MB)", "MemAlloc", "MemFreeMin"); + System.out.println("--------------------------------------------------------------------------------------------------------"); + + for (SinfoRecord r : resp.sinfo) { + System.out.printf(rowFmt, + r.partition.name, + r.nodes.total, + r.nodes.idle, + r.cpus.total, + r.cpus.idle, + r.node.state, + r.gres.total, + r.memory.maximum, + r.memory.allocated, + r.memory.free.minimum.number); + } + System.out.println("--------------------------------------------------------------------------------------------------------"); + System.out.printf("Slurm release: %s | Errors: %s%n", + resp.meta.slurm.release, + resp.errors); + } + + + + +} + + + diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java new file mode 100755 index 00000000..0e99e739 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java @@ -0,0 +1,84 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import java.util.HashMap; +@Deprecated +public class SlurmConfig { +// #!/bin/bash +//#SBATCH --job-name=duui +//#SBATCH --cpus-per-task=2 +// #SBATCH --time=1:00:00 +// +// PORT=9000 +// INNER=9714 +// IMG=/home/jd/again.sif +// +// apptainer exec "$IMG" \ +// sh -c "cd /usr/src/app && uvicorn textimager_duui_spacy:app --host 0.0.0.0 --port $INNER" & +// +// PID=$! +// socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER & +// PID_SOCAT=$! +// +// wait $PID +// kill $PID_SOCAT + + private static String slurmJobName = "slurmJobName"; + private static String slurmImagePort = "slurmPort"; + private static String slurmHostPort = "slurmHostPort"; + private static String slurmRuntime = "slurmRuntime"; + private static String slurmCpus = "slurmCpus"; + private static String slurmMemory = "slurmMemory"; + private static String slurmErrorLocation = "slurmErrorLocation"; + private static String slurmOutPutLocation = "slurmOutPutLocation"; + private static String slurmSIFSaveIn = "slurmSIFSaveIn"; + private static String slurmGPU = "slurmGPU"; + private static String slurmSIFImageName = "slurmSIFImageName"; + private static String slurmEntryLocation = "slurmEntryLocation"; + + + private String slurmJobNameV; + private String slurmImagePortV; + private String slurmHostPortV; + private String slurmRuntimeV; + private String slurmCpusV; + private String slurmMemoryV; + private String slurmErrorLocationV; + private String slurmOutPutLocationV; + private String slurmSIFSaveInV; + private String slurmGPUV; + private String slurmSIFImageNameV; + private String entryLocationV; + private HashMap slurmConfigProp = new HashMap(); + + + public SlurmConfig(String slurmJobNameV, String slurmImagePortV, String slurmHostPortV, String slurmRuntimeV, String slurmCpusV, String slurmMemoryV, String slurmErrorLocationV, String slurmOutPutLocationV, String slurmSIFSaveInV, String slurmGPUV, String slurmSIFImageNameV, String entryLocationV) { + this.slurmJobNameV = slurmJobNameV; + this.slurmImagePortV = slurmImagePortV; + this.slurmHostPortV = slurmHostPortV; + this.slurmRuntimeV = slurmRuntimeV; + this.slurmCpusV = slurmCpusV; + this.slurmMemoryV = slurmMemoryV; + this.slurmErrorLocationV = slurmErrorLocationV; + this.slurmOutPutLocationV = slurmOutPutLocationV; + this.slurmSIFSaveInV = slurmSIFSaveInV; + this.slurmGPUV = slurmGPUV; + this.slurmSIFImageNameV = slurmSIFImageNameV; + this.entryLocationV = entryLocationV; + //---------------------------------------------------------------- + slurmConfigProp.put(slurmJobName, slurmJobNameV); + slurmConfigProp.put(slurmImagePort, slurmImagePortV); + slurmConfigProp.put(slurmHostPort, slurmHostPortV); + slurmConfigProp.put(slurmRuntime, slurmRuntimeV); + slurmConfigProp.put(slurmCpus, slurmCpusV); + slurmConfigProp.put(slurmMemory, slurmMemoryV); + slurmConfigProp.put(slurmErrorLocation, slurmErrorLocationV); + slurmConfigProp.put(slurmOutPutLocation, slurmOutPutLocationV); + slurmConfigProp.put(slurmSIFSaveIn, slurmSIFSaveInV); + slurmConfigProp.put(slurmGPU, slurmGPUV); + slurmConfigProp.put(slurmSIFImageName, entryLocationV); + slurmConfigProp.put(slurmEntryLocation, entryLocationV); + + } + + +} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java new file mode 100755 index 00000000..1781faa1 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java @@ -0,0 +1,274 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.uima.cas.CASException; +import org.apache.uima.jcas.JCas; +import org.javatuples.Triplet; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.IDUUICommunicationLayer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.DUUIWebsocketAlt; +import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.IDUUIConnectionHandler; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.IDUUIInstantiatedPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.IDUUIUrlAccessible; +import org.texttechnologylab.DockerUnifiedUIMAInterface.pipeline_storage.DUUIPipelineDocumentPerformance; +import org.texttechnologylab.duui.ReproducibleAnnotation; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InvalidObjectException; +import java.net.ProxySelector; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; + +public class SlurmProcess { + public static HttpClient _client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .followRedirects(HttpClient.Redirect.ALWAYS) + .proxy(ProxySelector.getDefault()) + .connectTimeout(Duration.ofSeconds(1000)).build(); + + /** + * Calling the DUUI component + * @param jc + * @param comp + * @param perf + * @throws CompressorException + * @throws IOException + * @throws SAXException + * @throws CASException + */ + public static void process(JCas jc, IDUUIInstantiatedPipelineComponent comp, DUUIPipelineDocumentPerformance perf) throws CompressorException, IOException, SAXException, CASException { + + + + Triplet queue = comp.getComponent(); +// get lua script + IDUUICommunicationLayer layer = queue.getValue0().getCommunicationLayer(); + long serializeStart = System.nanoTime(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(1024*1024); + + DUUIPipelineComponent pipelineComponent = comp.getPipelineComponent(); + String viewName = pipelineComponent.getViewName(); + JCas viewJc; + if(viewName == null) { + viewJc = jc; + } + else { + try { + viewJc = jc.getView(viewName); + } + catch(CASException e) { + if(pipelineComponent.getCreateViewFromInitialView()) { + viewJc = jc.createView(viewName); + viewJc.setDocumentText(jc.getDocumentText()); + viewJc.setDocumentLanguage(jc.getDocumentLanguage()); + } + else { + throw e; + } + } + } + + layer.serialize(viewJc,out,comp.getParameters(), comp.getSourceView()); + // lua serialize call() + + byte[] ok = out.toByteArray(); + long sizeArray = ok.length; + long serializeEnd = System.nanoTime(); + + long annotatorStart = serializeEnd; + int tries = 0; + HttpResponse resp = null; + while (tries < 3) { + tries++; + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(queue.getValue0().generateURL() + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS)) + .timeout(Duration.ofSeconds(comp.getPipelineComponent().getTimeout())) + .POST(HttpRequest.BodyPublishers.ofByteArray(ok)) + .version(HttpClient.Version.HTTP_1_1) + .build(); + resp = _client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join(); + break; + } + catch(Exception e) { + e.printStackTrace(); + //System.out.printf("Cannot reach endpoint trying again %d/%d...\n",tries+1,10); + } + } + if(resp==null) { + throw new IOException("Could not reach endpoint after 3 tries!"); + } + + + if (resp.statusCode() == 200) { + ByteArrayInputStream st = new ByteArrayInputStream(resp.body()); + long annotatorEnd = System.nanoTime(); + long deserializeStart = annotatorEnd; + + try { + layer.deserialize(viewJc, st, comp.getTargetView()); + } + catch(Exception e) { + System.err.printf("Caught exception printing response %s\n",new String(resp.body(), StandardCharsets.UTF_8)); + + // TODO better error handleing flow? + comp.addComponent(queue.getValue0()); + + // TODO handle error docs for db here too? + + throw e; + } + long deserializeEnd = System.nanoTime(); + + ReproducibleAnnotation ann = new ReproducibleAnnotation(jc); + ann.setDescription(comp.getPipelineComponent().getFinalizedRepresentation()); + ann.setCompression(DUUIPipelineComponent.compressionMethod); + ann.setTimestamp(System.nanoTime()); + ann.setPipelineName(perf.getRunKey()); + ann.addToIndexes(); + perf.addData(serializeEnd-serializeStart,deserializeEnd-deserializeStart,annotatorEnd-annotatorStart,queue.getValue2()-queue.getValue1(),deserializeEnd-queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, null); + + comp.addComponent(queue.getValue0()); + } else { + comp.addComponent(queue.getValue0()); + ByteArrayInputStream st = new ByteArrayInputStream(resp.body()); + String responseBody = new String(st.readAllBytes(), StandardCharsets.UTF_8); + st.close(); + + // track "performance" of error documents if not explicitly disabled + if (perf.shouldTrackErrorDocs()) { + long annotatorEnd = System.nanoTime(); + long deserializeStart = annotatorEnd; + long deserializeEnd = System.nanoTime(); + + String error = "Expected response 200, got " + resp.statusCode() + ": " + responseBody; + + perf.addData(serializeEnd - serializeStart, deserializeEnd - deserializeStart, annotatorEnd - annotatorStart, queue.getValue2() - queue.getValue1(), deserializeEnd - queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, error); + } + + if (!pipelineComponent.getIgnoringHTTP200Error()) { + throw new InvalidObjectException(String.format("Expected response 200, got %d: %s", resp.statusCode(), responseBody)); + } else { + System.err.println(String.format("Expected response 200, got %d: %s", resp.statusCode(), responseBody)); + } + } + } + + /** + * The process merchant describes the use of the component as a web socket + * @param jc + * @param comp + * @param perf + * @throws CompressorException + * @throws IOException + * @throws SAXException + * @throws CASException + * @throws InterruptedException + */ + public static void process_handler(JCas jc, + IDUUIInstantiatedPipelineComponent comp, + DUUIPipelineDocumentPerformance perf) throws CompressorException, IOException, SAXException, CASException, InterruptedException { + Triplet queue = comp.getComponent(); + + IDUUICommunicationLayer layer = queue.getValue0().getCommunicationLayer(); + long serializeStart = System.nanoTime(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + DUUIPipelineComponent pipelineComponent = comp.getPipelineComponent(); + + String viewName = pipelineComponent.getViewName(); + JCas viewJc; + if(viewName == null) { + viewJc = jc; + } + else { + try { + viewJc = jc.getView(viewName); + } + catch(CASException e) { + if(pipelineComponent.getCreateViewFromInitialView()) { + viewJc = jc.createView(viewName); + viewJc.setDocumentText(jc.getDocumentText()); + viewJc.setDocumentLanguage(jc.getDocumentLanguage()); + } + else { + throw e; + } + } + } + // lua serialize call() + layer.serialize(viewJc,out,comp.getParameters(), comp.getSourceView()); + + // ok is the message. + byte[] ok = out.toByteArray(); + long sizeArray = ok.length; + long serializeEnd = System.nanoTime(); + + long annotatorStart = serializeEnd; + + /** + * @edited Givara Ebo, Dawit Terefe + * + * Retrieve websocket-client from IDUUIUrlAccessible (ComponentInstance). + * + */ + IDUUIUrlAccessible accessible = queue.getValue0(); + IDUUIConnectionHandler handler = accessible.getHandler(); + + if (handler.getClass() == DUUIWebsocketAlt.class){ + String error = null; + + JCas finalViewJc = viewJc; + + List results = handler.send(ok); + + long annotatorEnd = System.nanoTime(); + long deserializeStart = annotatorEnd; + + ByteArrayInputStream result = null; + try { + /*** + * @edited + * Givara Ebo, Dawit Terefe + * + * Merging results before deserializing. + */ + result = layer.merge(results); + layer.deserialize(finalViewJc, result, comp.getTargetView()); + } + catch(Exception e) { + e.printStackTrace(); + System.err.printf("Caught exception printing response %s\n",new String(result.readAllBytes(), StandardCharsets.UTF_8)); + + // TODO more error handling needed? + error = ExceptionUtils.getStackTrace(e); + } + + long deserializeEnd = System.nanoTime(); + + comp.addComponent(accessible); + + ReproducibleAnnotation ann = new ReproducibleAnnotation(jc); + ann.setDescription(comp.getPipelineComponent().getFinalizedRepresentation()); + ann.setCompression(DUUIPipelineComponent.compressionMethod); + ann.setTimestamp(System.nanoTime()); + ann.setPipelineName(perf.getRunKey()); + ann.addToIndexes(); + perf.addData(serializeEnd-serializeStart,deserializeEnd-deserializeStart,annotatorEnd-annotatorStart,queue.getValue2()-queue.getValue1(),deserializeEnd-queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, error); + comp.addComponent(accessible); + } + } + +} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java new file mode 100755 index 00000000..fea80fd2 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java @@ -0,0 +1,272 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SlurmUtils { +// ein paar cli command in java umschreiben/kaspseln + public static boolean isMasterNode(){ + String s = whoIsMaster().split("=", 2)[1]; + return s.equals(whoAmI());} + + public static String whoIsMaster(){ + try { + InputStream pb = new ProcessBuilder("bash", "-c", "cat /etc/slurm/slurm.conf | grep ControlMachine").start().getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); + return reader.readLine(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static String whoAmI(){ + try { + InputStream pb = new ProcessBuilder("hostname").start().getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); + return reader.readLine(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static boolean pullSifImagefromRemoteDockerRepo(String imagePath, String imageName) { + if (imagePath == null || imagePath.isEmpty()) { + throw new IllegalArgumentException("imagePath is null or empty"); + } + + System.out.println("Correct format: apptainer build {Sif_image_name}.sif docker://{docker_repo/image:tag}"); + + List command = List.of( + "apptainer", "build", + imageName + ".sif", + "docker://" + imagePath + ); + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + pb.inheritIO(); + try { + Process process = pb.start(); + int exitCode = process.waitFor(); + return exitCode == 0; + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to run apptainer build", e); + } + } + + + public static boolean pullSifImagefromLocalDockerRepo(String imagePath, String imageName) { + if (imagePath == null || imagePath.isEmpty()) { + throw new IllegalArgumentException("imagePath is null or empty"); + } + + System.out.println("Correct format: apptainer build {Sif_image_name}.sif docker-daemon://{image:tag}"); + + List command = List.of( + "apptainer", "build", + imageName + ".sif", + "docker-daemon://" + imagePath + ); + + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + pb.inheritIO(); + try { + Process process = pb.start(); + int exitCode = process.waitFor(); + return exitCode == 0; + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to run apptainer build", e); + } + } + + + public static boolean submitHeldJob(String scriptPath) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("sbatch", "--hold", scriptPath); + pb.redirectErrorStream(true); + Process process = pb.start(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("Submitted batch job")) { + return true; + } + } + } + + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("sbatch submission failed"); + } + return false; + } + + +// public static void releaseJob(String jobId) throws IOException, InterruptedException { +// ProcessBuilder pb = new ProcessBuilder("scontrol", "release", jobId); +// pb.environment().put("SLURM_JOB_ID", jobId); +// pb.inheritIO(); +// Process process = pb.start(); +// int exitCode = process.waitFor(); +// if (exitCode != 0) { +// throw new RuntimeException("Failed to release job " + jobId); +// } +// } + + + public static List getHeldJobs() throws IOException, InterruptedException { + List jobIds = new ArrayList<>(); + ProcessBuilder pb = new ProcessBuilder("squeue", "--user=" + System.getProperty("user.name"), "--state=PD", "-h", "-o", "%i"); + Process process = pb.start(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + jobIds.add(Integer.parseInt(line.trim())); + } + } + + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Failed to fetch held jobs"); + } + + return jobIds; + } +// +// public static void releaseAllHeldJobs() throws IOException, InterruptedException { +// List heldJobs = getHeldJobs(); +// for (int jobId : heldJobs) { +// releaseJob(jobId); +// } +// } +// + + + + public static void squeue() throws IOException { + ProcessBuilder pb = new ProcessBuilder("squeue"); + pb.inheritIO(); + pb.start(); + } + + + public static void cancelJob(int jobId) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("scancel", String.valueOf(jobId)); + pb.inheritIO(); + Process process = pb.start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("Failed to cancel job " + jobId); + } + } + + + + + // Ich finde das ziemlich zeitaufwendig, deshalb möchte ich später eine ähnliche Webseite für duui entwickeln + //Zum Beispiel könnte man den Namen einer DUUI-Komponente eingeben, und das System würde automatisch eine passende + // Konfiguration mit Entrypoint vorschlagen oder vervollständigen. + public static String submitJob(String jobName,String hostport, + String cpus,String gpu,String entry, String error, String imageport,String mem, + String time, + String output,String sifname, String saveIn, String uvicorn + ) throws IOException, InterruptedException { + + Path script = Paths.get("/tmp", jobName + ".sh"); + + + List lines = List.of( + "#!/bin/bash", + "#SBATCH --job-name=" + jobName, + "#SBATCH --cpus-per-task=" + cpus, + "#SBATCH --time=" + time, + "", + "PORT=" + hostport, + "UVI=\"" + uvicorn + "\"", // + "INNER=" + imageport, + "IMG=\"" + saveIn + "\"", + "INTOIMAGE=\"" + entry + "\"", // looks like "cd /usr/src/app" + "", + "apptainer exec \"$IMG\" \\", + " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &", + "PID=$!", + "", + "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &", + "PID_SOCAT=$!", + "", + "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT", + "", + "wait $PID" + ); + + + Files.write(script, lines, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + script.toFile().setExecutable(true); + System.out.println("Slurm batch script written to: " + script); + + + ProcessBuilder pb = new ProcessBuilder("sbatch", "--parsable",script.toString()); + pb.redirectErrorStream(true); + Process proc = pb.start(); + + String jobId; + try (BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()))) { + jobId = br.readLine(); + } + int exit = proc.waitFor(); + if (exit != 0 || jobId == null || jobId.isBlank()) { + throw new IllegalStateException("sbatch failed，exit=" + exit); + } + + System.out.println("Job submitted. ID = " + jobId); + return jobId.trim(); + } + + public static boolean checkSocatInstalled(){ + try { + InputStream pb = new ProcessBuilder("which", "socat").start().getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); + String s = reader.readLine(); + System.out.println(s); + return !s.isEmpty(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static boolean checkSlurmInstalled(){ + try { + InputStream pb = new ProcessBuilder("slurmd", " ", "-C").start().getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); + String s = reader.readLine(); + System.out.println(s); + return !s.isEmpty(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + + + + + +} diff --git a/src/test/java/SlurmTests.java b/src/test/java/SlurmTests.java new file mode 100644 index 00000000..694a5604 --- /dev/null +++ b/src/test/java/SlurmTests.java @@ -0,0 +1,68 @@ +import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; +import org.apache.uima.fit.factory.JCasFactory; +import org.apache.uima.jcas.JCas; +import org.dkpro.core.io.xmi.XmiWriter; +import org.junit.jupiter.api.Test; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaSandbox; + +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; + +public class SlurmTests { + @Test + public void basicTest() throws Exception { + int iWorkers = 1; // + JCas jc = JCasFactory.createText("hello world"); + DocumentMetaData dmd = new DocumentMetaData(jc); + dmd.setDocumentId("test05"); + dmd.setDocumentTitle("univers test05"); + dmd.addToIndexes(); + DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); + DUUILuaSandbox sandbox = new DUUILuaSandbox(); + sandbox._allowAllJavaClasses = true; + ctx.withSandbox(sandbox); + DUUIComposer composer = new DUUIComposer() + .withSkipVerification(true) + .withLuaContext(ctx) + .withWorkers(iWorkers); + DUUIUIMADriver uimaDriver = new DUUIUIMADriver(); + // driver hinzufügen + DUUISlurmDriver slurmDriver = new DUUISlurmDriver(); + + composer.addDriver(uimaDriver, slurmDriver); + + composer.resetPipeline(); + DUUIPipelineComponent com1 = new DUUISlurmDriver.Component(new DUUIPipelineComponent() + .withSlurmCpus("2") + .withSlurmGPU("0") + .withSlurmMemory("1G") + .withSlurmImagePort("9714") + .withSlurmJobName("test1") + .withSlurmEntryLocation("cd /usr/src/app") + .withSlurmErrorLocation("/tmp") + .withSlurmSaveIn("/home/jd/again.sif") + .withSlurmRuntime("1:00:00") + .withScale(1) + .withSlurmUvicorn("uvicorn textimager_duui_spacy:app") + ).build(); + + + composer.add(com1); + + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1", + XmiWriter.PARAM_COMPRESSION, "GZIP")) + .build()); + + composer.run(jc); + + + } +} From c71c6df3be612222e5dd0b0e288e2592aa07bc44 Mon Sep 17 00:00:00 2001 From: Jiadong Zhou Date: Mon, 9 Jun 2025 20:36:02 +0200 Subject: [PATCH 2/5] remove .idea from git --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0c733b49..621195b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -target/* -.idea/* +target/ +.idea/ *.png *.pgf *.db From 712085e62c2090d1f1ab004f5d82dde6387a56eb Mon Sep 17 00:00:00 2001 From: Jiadong Zhou Date: Mon, 9 Jun 2025 20:42:55 +0200 Subject: [PATCH 3/5] change version, sync with upstream --- pom.xml | 4 +-- src/.idea/misc.xml | 6 ----- src/.idea/modules.xml | 10 -------- src/.idea/vcs.xml | 6 ----- src/.idea/workspace.xml | 54 ----------------------------------------- 5 files changed, 2 insertions(+), 78 deletions(-) delete mode 100644 src/.idea/misc.xml delete mode 100644 src/.idea/modules.xml delete mode 100644 src/.idea/vcs.xml delete mode 100644 src/.idea/workspace.xml diff --git a/pom.xml b/pom.xml index 65274e3b..38017a0a 100755 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.texttechnologylab DockerUnifiedUIMAInterface - 1.4.6 + 1.5.0 Automatic analysis of large text corpora is a complex task. This complexity particularly concerns the question of time efficiency. Furthermore, efficient, flexible, and extensible textanalysis requires the continuous integration of every new text analysis tools. Since there are currently, in the area of NLP and especially in the application context of UIMA, only very few to no adequate frameworks for these purposes, which are not simultaneously outdated or can no longer be used for security reasons, this work will present a new approach to fill this gap. To this end, we present Docker Unified UIMA Interface (DUUI), a scalable, flexible, lightweight, and featurerich framework for automated and distributed analysis of text corpora that leverages experience in Big Data analytics and virtualization with Docker. @@ -77,7 +77,7 @@ UTF-8 21 21 - 3.0.7 + 3.0.8 3.0.0 3.5.0 2.4.0 diff --git a/src/.idea/misc.xml b/src/.idea/misc.xml deleted file mode 100644 index 47478b91..00000000 --- a/src/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/src/.idea/modules.xml b/src/.idea/modules.xml deleted file mode 100644 index 31eaf94d..00000000 --- a/src/.idea/modules.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/src/.idea/vcs.xml b/src/.idea/vcs.xml deleted file mode 100644 index 6c0b8635..00000000 --- a/src/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/src/.idea/workspace.xml b/src/.idea/workspace.xml deleted file mode 100644 index f1bdb6ee..00000000 --- a/src/.idea/workspace.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - 1749492335426 - - - - - - \ No newline at end of file From f2175d4cfb7556d8cb72935605eeda2788349eac Mon Sep 17 00:00:00 2001 From: Jiadong Zhou Date: Mon, 9 Jun 2025 20:54:19 +0200 Subject: [PATCH 4/5] PortManager should be replaced or cooperate with ssh --- .../driver/slurm/DUUISlurmDriver.java | 2 +- .../driver/slurm/Portmanager.java | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java index 76677d95..86f0216b 100755 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java @@ -262,7 +262,7 @@ public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skip // if (!(component.getSlurmSIFRepoLocation() == null)) { // // mean local no sif file // //_docker_interface.pullImage(component.getDockerImageName()); -// // 自动把sif iamge name 传入 dockerimagename +// sif iamge name in dockerimagename // SlurmUtils.pullSifImagefromRemoteDockerRepo(component.getSlurmSIFImageName(), component.getDockerImageName()); // } // // diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java index 52ff9c1a..e2e8f719 100755 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java @@ -7,32 +7,42 @@ import java.time.Duration; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - +// should use ssh -L later /* 20000–30000 speiziell reserviert fuer slurm */ final class PortManager { private static final int BASE = 20000; private static final int MAX = 30000; private static final Set RESERVED = ConcurrentHashMap.newKeySet(); - /** get a free port for placeholder -> make it RESERVED */ + /** + * get a free port for placeholder -> make it RESERVED + */ static int acquire() throws IOException { for (int p = BASE; p < MAX; p++) { - if (!RESERVED.add(p)) continue; // conflict - if (isFree(p)) return p; // success + if (!RESERVED.add(p)) { + continue; + } // conflict + if (isFree(p)) { + return p; + } // success RESERVED.remove(p); // bind failed } throw new IOException("No free port in pool"); } - static void release(int port) { RESERVED.remove(port); } + static void release(int port) { + RESERVED.remove(port); + } private static boolean isFree(int port) { try (ServerSocket s = new ServerSocket()) { s.setReuseAddress(false); s.bind(new InetSocketAddress("0.0.0.0", port)); return true; - } catch (IOException e) { return false; } + } catch (IOException e) { + return false; + } } // ich vergesse immer am ende close()anzurufen, benuztze autocloseable, anweisungen in {} verklammern @@ -46,7 +56,11 @@ static class PortReservation implements AutoCloseable { this.placeholder.setReuseAddress(false); this.placeholder.bind(new InetSocketAddress("0.0.0.0", port)); } - @Override public void close() throws IOException { placeholder.close(); } + + @Override + public void close() throws IOException { + placeholder.close(); + } } // schickt ein paar @@ -54,20 +68,18 @@ static void waitUntilReachable(String host, int port, Duration timeout) throws IOException, InterruptedException { long deadline = System.nanoTime() + timeout.toNanos(); - long backoff = 250; // ms!!! + long backoff = 250; // ms!!! while (System.nanoTime() < deadline) { try (Socket s = new Socket()) { s.connect(new InetSocketAddress(host, port), 1000); return; - } catch (IOException ignore) { } + } catch (IOException ignore) { + } Thread.sleep(backoff); backoff = Math.min(backoff * 2, 2000); } - throw new IOException("Port "+port+" did not open within "+timeout.toSeconds()+" s"); + throw new IOException("Port " + port + " did not open within " + timeout.toSeconds() + " s"); } - - - } From 075a0c952d0b864c22f3e5a24640f3afa12f6727 Mon Sep 17 00:00:00 2001 From: Jiadong Zhou Date: Tue, 22 Jul 2025 21:08:59 +0200 Subject: [PATCH 5/5] slurm in docker driver 1.0 done --- pom.xml | 156 +++--- .../DUUIComposer.java | 6 +- .../driver/DUUIPipelineComponent.java | 443 ++++++++-------- .../driver/IDUUIDriverInterface.java | 4 +- .../driver/slurm/DUUISlurmDriver.java | 476 ++++++------------ .../driver/slurm/DUUISlurmInterface.java | 129 +++-- .../driver/slurm/PortManager.java | 108 ++++ .../driver/slurm/Portmanager.java | 85 ---- .../driver/slurm/SinfoData.java | 149 ------ .../driver/slurm/SlurmConfig.java | 84 ---- .../driver/slurm/SlurmProcess.java | 274 ---------- .../driver/slurm/SlurmUtils.java | 335 +++++------- .../driver/slurm/slurmInDocker/SlurmRest.java | 237 +++++++++ src/test/java/SlurmTests.java | 68 --- .../io/abbyy/TestAbbyyDocumentReader.java | 2 +- .../lua_process/TestGNFinderLite.java | 260 +++++----- .../lua_process/TestSpacyBatched.java | 292 +++++------ .../lua_process/TestSpacyEos.java | 176 +++---- .../test/DockerTestContainerManager.java | 95 ---- .../tools/TestAnnotationDropper.java | 2 +- .../slurmDriver/ParlbertTopicGermanTest.java | 147 ++++++ src/test/java/slurmDriver/RestApi.java | 348 +++++++++++++ src/test/java/slurmDriver/universTest.java | 96 ++++ 23 files changed, 1978 insertions(+), 1994 deletions(-) mode change 100644 => 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java delete mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java delete mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java delete mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java delete mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java create mode 100755 src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java delete mode 100644 src/test/java/SlurmTests.java delete mode 100644 src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java create mode 100755 src/test/java/slurmDriver/ParlbertTopicGermanTest.java create mode 100755 src/test/java/slurmDriver/RestApi.java create mode 100755 src/test/java/slurmDriver/universTest.java diff --git a/pom.xml b/pom.xml index 38017a0a..7c12a0e0 100755 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.texttechnologylab DockerUnifiedUIMAInterface - 1.5.0 + 1.4.6 Automatic analysis of large text corpora is a complex task. This complexity particularly concerns the question of time efficiency. Furthermore, efficient, flexible, and extensible textanalysis requires the continuous integration of every new text analysis tools. Since there are currently, in the area of NLP and especially in the application context of UIMA, only very few to no adequate frameworks for these purposes, which are not simultaneously outdated or can no longer be used for security reasons, this work will present a new approach to fill this gap. To this end, we present Docker Unified UIMA Interface (DUUI), a scalable, flexible, lightweight, and featurerich framework for automated and distributed analysis of text corpora that leverages experience in Big Data analytics and virtualization with Docker. @@ -77,7 +77,7 @@ UTF-8 21 21 - 3.0.8 + 3.0.7 3.0.0 3.5.0 2.4.0 @@ -141,49 +141,49 @@ full - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins maven-compiler-plugin @@ -259,7 +259,7 @@ ${uima.version} - + org.apache.httpcomponents.client5 httpclient5 @@ -290,17 +290,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -371,7 +371,7 @@ com.influxdb influxdb-client-java - + 6.9.0 @@ -484,25 +484,25 @@ compile - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + @@ -535,7 +535,7 @@ 8.5.6 - + io.fabric8 kubernetes-client 6.8.0 diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIComposer.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIComposer.java index 47b1fe73..ef8d9f97 100644 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIComposer.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/DUUIComposer.java @@ -829,6 +829,10 @@ public DUUIComposer() throws URISyntaxException { addEvent(DUUIEvent.Sender.COMPOSER, "Shutdown Hook finished."); } catch (UnknownHostException e) { e.printStackTrace(); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); } }); @@ -1932,7 +1936,7 @@ public int getWorkerCount() { * Shuts down the DUUI controller by signaling every worker and stopping all components. * @throws UnknownHostException */ - public void shutdown() throws UnknownHostException { + public void shutdown() throws IOException, InterruptedException { if (isService) { addEvent( DUUIEvent.Sender.COMPOSER, diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java old mode 100644 new mode 100755 index e9eb7509..e8f237a3 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/DUUIPipelineComponent.java @@ -24,17 +24,27 @@ /** * Encapsulation of a component for a pipeline. + * * @author Alexander Leonhardt */ + + public class DUUIPipelineComponent { + private HashMap _options; - private HashMap _parameters; + + private HashMap _parameters; + private AnalysisEngineDescription _engine; + private String _finalizedEncoded; + private int _finalizedEncodedHash; private String _compression; + private boolean _websocket = false; + private int _ws_elements = 50; private List _constraints = new ArrayList<>(0); @@ -74,20 +84,23 @@ public class DUUIPipelineComponent { private static String timeout = "timeout"; private static String slurmJobName = "slurmJobName"; - private static String slurmImagePort = "slurmPort"; - //private static String slurmHostPort = "slurmHostPort"; + private static String slurmHostPort = "slurmHostPort"; private static String slurmRuntime = "slurmRuntime"; private static String slurmCpus = "slurmCpus"; private static String slurmMemory = "slurmMemory"; private static String slurmErrorLocation = "slurmErrorLocation"; private static String slurmOutPutLocation = "slurmOutPutLocation"; - private static String slurmSIFDiskLocation = "slurmSIFDiskLocation"; - private static String slurmSIFRepoLocation = "slurmSIFRepoLocation"; + private static String slurmSIFLocation = "slurmSIFLocation"; private static String slurmGPU = "slurmGPU"; private static String slurmSIFImageName = "slurmSIFImageName"; - private static String slurmEntryLocation = "slurmEntryLocation"; private static String slurmNoShutdown = "slurmNoShutdown"; private static String slurmUvicorn = "slurmUvicorn"; + private static String slurmScript = "slurmScript"; + private static String slurmPartition = "slurmPartition"; + private static String slurmNodelist = "slurmNodelist"; + private static String slurmWorkDir = "slurmWorkDir"; + private static String slurmInnerPort = "slurmInnerPort"; + private String getVersion() throws URISyntaxException, IOException { ClassLoader classLoader = DUUIPipelineComponent.class.getClassLoader(); try { @@ -99,31 +112,34 @@ private String getVersion() throws URISyntaxException, IOException { } public DUUIPipelineComponent() throws URISyntaxException, IOException { + _options = new HashMap<>(); + _finalizedEncoded = null; _parameters = new HashMap<>(); String version = getVersion(); - if(version == null) { - _options.put(versionInformation,"Unknown"); - } - else { - _options.put(versionInformation,version); + if (version == null) { + _options.put(versionInformation, "Unknown"); + } else { + _options.put(versionInformation, version); } _parameters.put(websocketOptionName, String.valueOf(_websocket)); } public void finalizeComponent() throws CompressorException, IOException, SAXException { - if(_engine!=null) { + if (_engine != null) { StringWriter writer = new StringWriter(); _engine.toXML(writer); _options.put(engineOptionName, writer.getBuffer().toString()); } ByteArrayOutputStream out = new ByteArrayOutputStream(); - CompressorOutputStream cos = new CompressorStreamFactory().createCompressorOutputStream(compressionMethod,out); + + CompressorOutputStream cos = new CompressorStreamFactory().createCompressorOutputStream(compressionMethod, out); cos.write(toJson().getBytes(StandardCharsets.UTF_8)); cos.close(); + _finalizedEncoded = Base64.getEncoder().encodeToString(out.toByteArray()); _finalizedEncodedHash = _finalizedEncoded.hashCode(); } @@ -137,19 +153,21 @@ public int getFinalizedRepresentationHash() { } public DUUIPipelineComponent withDriver(Class t) { - _options.put(driverName,t.getCanonicalName()); + _options.put(driverName, t.getCanonicalName()); return this; } + public String getDriver() { return _options.get(driverName); } + public DUUIPipelineComponent withDescription(String description) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(description == null) { + if (description == null) { _options.remove(descriptionName); return this; } @@ -157,16 +175,17 @@ public DUUIPipelineComponent withDescription(String description) { return this; } + public String getDescription() { return _options.get(descriptionName); } public DUUIPipelineComponent withName(String name) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(name == null) { + if (name == null) { _options.remove(componentName); return this; } @@ -180,16 +199,16 @@ public String getName() { public String getName(String defaultValue) { String value = _options.get(componentName); - if(value==null) return defaultValue; + if (value == null) return defaultValue; return value; } public DUUIPipelineComponent withEngine(AnalysisEngineDescription desc) throws IOException, SAXException { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } _engine = desc; - if(desc == null) { + if (desc == null) { _options.remove(engineOptionName); return this; } @@ -197,10 +216,10 @@ public DUUIPipelineComponent withEngine(AnalysisEngineDescription desc) throws I } public AnalysisEngineDescription getEngine() throws IOException, SAXException, InvalidXMLException { - if(_engine!=null) return _engine; + if (_engine != null) return _engine; String engine = _options.get(engineOptionName); - if(engine==null) return null; + if (engine == null) return null; String temp = Files.createTempFile("duuid_driver_uima", ".xml").toFile().getAbsolutePath(); Files.write(Paths.get(temp), engine.getBytes(StandardCharsets.UTF_8)); @@ -208,8 +227,9 @@ public AnalysisEngineDescription getEngine() throws IOException, SAXException, I return _engine; } + // public DUUIPipelineComponent withScale(Integer iScale) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } @@ -241,7 +261,7 @@ public DUUIPipelineComponent withConstraint(String constraints) { return this; } - public List getConstraints(){ + public List getConstraints() { return _constraints; } @@ -251,65 +271,65 @@ public Integer getScale() { public Integer getScale(Integer defaultValue) { String scale = _options.get(scaleOptionName); - if(scale == null) return defaultValue; + if (scale == null) return defaultValue; return Integer.parseInt(scale); } public DUUIPipelineComponent withUrl(String url) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(url == null) { + if (url == null) { _options.remove(urlOptionName); return this; } JSONArray js = new JSONArray(); js.put(url); - _options.put(urlOptionName,js.toString()); + _options.put(urlOptionName, js.toString()); return this; } public DUUIPipelineComponent withUrls(List urls) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(urls == null) { + if (urls == null) { _options.remove(urlOptionName); return this; } JSONArray arr = new JSONArray(); - for(String s : urls) { + for (String s : urls) { arr.put(s); } - _options.put(urlOptionName,arr.toString()); + _options.put(urlOptionName, arr.toString()); return this; } public List getUrl() { LinkedList lst = new LinkedList<>(); String urls = _options.get(urlOptionName); - if(urls == null) return null; + if (urls == null) return null; JSONArray arr = new JSONArray(urls); - for(int i = 0; i < arr.length(); i++) { + for (int i = 0; i < arr.length(); i++) { lst.push(arr.getString(i)); } return lst; } public DUUIPipelineComponent withDockerAuth(String name, String password) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(name==null && password == null) { + if (name == null && password == null) { _options.remove(dockerPasswordOptionName); _options.remove(dockerUsernameOptionName); return this; } - _options.put(dockerPasswordOptionName,password); - _options.put(dockerUsernameOptionName,name); + _options.put(dockerPasswordOptionName, password); + _options.put(dockerUsernameOptionName, name); return this; } @@ -319,7 +339,7 @@ public String getDockerAuthUsername() { public String getDockerAuthUsername(String defaultValue) { String value = _options.get(dockerUsernameOptionName); - if(value == null) return defaultValue; + if (value == null) return defaultValue; return value; } @@ -329,20 +349,20 @@ public String getDockerAuthPassword() { public String getDockerAuthPassword(String defaultValue) { String value = _options.get(dockerPasswordOptionName); - if(value == null) return defaultValue; + if (value == null) return defaultValue; return value; } public DUUIPipelineComponent withDockerRunAfterExit(Boolean b) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(b==null) { + if (b == null) { _options.remove(dockerNoShutdown); return this; } - _options.put(dockerNoShutdown,String.valueOf(b)); + _options.put(dockerNoShutdown, String.valueOf(b)); return this; } @@ -352,20 +372,20 @@ public Boolean getDockerRunAfterExit() { public Boolean getDockerRunAfterExit(Boolean defaultValue) { String result = _options.get(dockerNoShutdown); - if(result == null) return defaultValue; + if (result == null) return defaultValue; return Boolean.parseBoolean(result); } public DUUIPipelineComponent withDockerGPU(Boolean b) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(b==null) { + if (b == null) { _options.remove(dockerWithGPU); return this; } - _options.put(dockerWithGPU,String.valueOf(b)); + _options.put(dockerWithGPU, String.valueOf(b)); return this; } @@ -375,31 +395,31 @@ public Boolean getDockerGPU() { public Boolean getDockerGPU(Boolean defaultValue) { String result = _options.get(dockerWithGPU); - if(result == null) return defaultValue; + if (result == null) return defaultValue; return Boolean.parseBoolean(result); } public DUUIPipelineComponent withDockerImageName(String imageName) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(imageName==null) { + if (imageName == null) { _options.remove(dockerImageName); return this; } - _options.put(dockerImageName,imageName); + _options.put(dockerImageName, imageName); return this; } public DUUIPipelineComponent __internalPinDockerImage(String imageName, String pinName) { - if(pinName==null) { + if (pinName == null) { System.err.println("Could not add the digest since this image has not been pushed and pulled from a registry V2"); - _options.put(dockerImageName,imageName); + _options.put(dockerImageName, imageName); return this; } - _options.put(dockerImageName,pinName); + _options.put(dockerImageName, pinName); return this; } @@ -409,22 +429,22 @@ public String getDockerImageName() { public String getDockerImageName(String defaultValue) { String result = _options.get(dockerImageName); - if(result == null) { + if (result == null) { return defaultValue; } return result; } public DUUIPipelineComponent withDockerImageFetching(Boolean b) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(b==null) { + if (b == null) { _options.remove(dockerImageFetching); return this; } - _options.put(dockerImageFetching,String.valueOf(b)); + _options.put(dockerImageFetching, String.valueOf(b)); return this; } @@ -434,32 +454,32 @@ public Boolean getDockerImageFetching() { public Boolean getDockerImageFetching(Boolean defaultValue) { String result = _options.get(dockerImageFetching); - if(result == null) return defaultValue; + if (result == null) return defaultValue; return Boolean.parseBoolean(result); } public DUUIPipelineComponent withWriteToView(String viewName) { - return withWriteToView(viewName,false); + return withWriteToView(viewName, false); } public DUUIPipelineComponent withWriteToView(String viewName, boolean createViewFromInitialView) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if(viewName==null) { + if (viewName == null) { _options.remove(writeToViewName); _options.remove(initialViewFromInitialViewName); return this; } - _options.put(writeToViewName,viewName); - _options.put(initialViewFromInitialViewName,String.valueOf(createViewFromInitialView)); + _options.put(writeToViewName, viewName); + _options.put(initialViewFromInitialViewName, String.valueOf(createViewFromInitialView)); return this; } public Boolean getCreateViewFromInitialView() { String value = _options.get(initialViewFromInitialViewName); - if(value == null) return null; + if (value == null) return null; return Boolean.valueOf(value); } @@ -469,34 +489,37 @@ public String getViewName() { public String getViewName(String defaultValue) { String value = _options.get(writeToViewName); - if(value==null) return defaultValue; + if (value == null) return defaultValue; return value; } public String toJson() { JSONObject js = new JSONObject(); - js.put("options",_options); - js.put("parameters",_parameters); + js.put("options", _options); + js.put("parameters", _parameters); return js.toString(); } /** * @author Dawit Terefe (edited) - * + *

* Option to choose websocket as protocol. * Default is false. * Option to choose number of elements for * partition size. - * */ - public boolean isWebsocket() { return _websocket; } + public boolean isWebsocket() { + return _websocket; + } public DUUIPipelineComponent withWebsocket(boolean b) { _websocket = b; return withParameter(websocketOptionName, String.valueOf(b)); } - public int getWebsocketElements () { return _ws_elements; } + public int getWebsocketElements() { + return _ws_elements; + } public DUUIPipelineComponent withWebsocket(boolean b, int elements) { _websocket = b; @@ -506,11 +529,11 @@ public DUUIPipelineComponent withWebsocket(boolean b, int elements) { } public DUUIPipelineComponent withParameter(String key, String value) { - _parameters.put(key,value); + _parameters.put(key, value); return this; } - public DUUIPipelineComponent withView(String viewName){ + public DUUIPipelineComponent withView(String viewName) { withSourceView(viewName); withTargetView(viewName); return this; @@ -535,18 +558,18 @@ public DUUIPipelineComponent withTimeout(long lLong) { public static DUUIPipelineComponent fromJson(String json) throws URISyntaxException, IOException { JSONObject jobj = new JSONObject(json); - HashMap optionsMap = new HashMap<>(); + HashMap optionsMap = new HashMap<>(); JSONObject options = jobj.getJSONObject("options"); for (Iterator it = options.keys(); it.hasNext(); ) { String key = it.next(); - optionsMap.put(key,options.getString(key)); + optionsMap.put(key, options.getString(key)); } - HashMap parametersMap = new HashMap<>(); + HashMap parametersMap = new HashMap<>(); JSONObject parameters = jobj.getJSONObject("parameters"); for (Iterator it = parameters.keys(); it.hasNext(); ) { String key = it.next(); - parametersMap.put(key,parameters.getString(key)); + parametersMap.put(key, parameters.getString(key)); } DUUIPipelineComponent comp = new DUUIPipelineComponent(); @@ -556,58 +579,54 @@ public static DUUIPipelineComponent fromJson(String json) throws URISyntaxExcept } public static DUUIPipelineComponent fromEncodedJson(String encodedJson) throws URISyntaxException, IOException, CompressorException { - return fromEncodedJson(encodedJson,compressionMethod); + return fromEncodedJson(encodedJson, compressionMethod); } public static DUUIPipelineComponent fromEncodedJson(String encodedJson, String compressionUsed) throws URISyntaxException, IOException, CompressorException { byte[] decoded = Base64.getDecoder().decode(encodedJson.getBytes(StandardCharsets.UTF_8)); - CompressorInputStream stream = new CompressorStreamFactory().createCompressorInputStream(compressionUsed,new ByteArrayInputStream(decoded)); - String json = new String(stream.readAllBytes(),StandardCharsets.UTF_8); + CompressorInputStream stream = new CompressorStreamFactory().createCompressorInputStream(compressionUsed, new ByteArrayInputStream(decoded)); + String json = new String(stream.readAllBytes(), StandardCharsets.UTF_8); return fromJson(json); } public DUUIPipelineComponent join(DUUIPipelineComponent comp, DUUIPipelineComponentJoinStrategy strategy) { - if(_finalizedEncoded!=null) { + if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - - if(strategy == DUUIPipelineComponentJoinStrategy.LEFT_WINS) { - for(String i : comp._options.keySet()) { + if (strategy == DUUIPipelineComponentJoinStrategy.LEFT_WINS) { + for (String i : comp._options.keySet()) { //Discard keys we have too write others into us - if(!_options.containsKey(i)) { - _options.put(i,comp._options.get(i)); + if (!_options.containsKey(i)) { + _options.put(i, comp._options.get(i)); } } - } - else if(strategy == DUUIPipelineComponentJoinStrategy.RIGHT_WINS) { + } else if (strategy == DUUIPipelineComponentJoinStrategy.RIGHT_WINS) { //Write everything since we loose on conflict and need missing options anyways - for(String i : comp._options.keySet()) { - _options.put(i,comp._options.get(i)); + for (String i : comp._options.keySet()) { + _options.put(i, comp._options.get(i)); } - } - else { - for(String i : comp._options.keySet()) { - if(_options.containsKey(i)) { - if(!_options.get(i).equals(comp._options.get(i))) { + } else { + for (String i : comp._options.keySet()) { + if (_options.containsKey(i)) { + if (!_options.get(i).equals(comp._options.get(i))) { throw new RuntimeException(String.format("Key conflict in join in key %s", i)); } - } - else { - _options.put(i,comp._options.get(i)); + } else { + _options.put(i, comp._options.get(i)); } } } return this; } - public final Map getParameters() { + public final Map getParameters() { return _parameters; } public String getSourceView() { String result = _options.get(sourceView); - if(result == null) { + if (result == null) { return "_InitialView"; } return result; @@ -615,7 +634,7 @@ public String getSourceView() { public String getTargetView() { String result = _options.get(targetView); - if(result == null) { + if (result == null) { return "_InitialView"; } return result; @@ -647,25 +666,21 @@ public DUUIUIMADriver.Component asUIMADriverComponent() throws URISyntaxExceptio } public String attemptAutomaticDescription() throws InvalidXMLException, URISyntaxException, IOException, SAXException { - if(getDriver().equals(DUUIUIMADriver.class.getCanonicalName())) { - return String.format("UIMA annotator: %s, scale: %d",asUIMADriverComponent().getAnnotatorName(),getScale(1)); - } - else if(getDriver().equals(DUUIDockerDriver.class.getCanonicalName())) { - return String.format("Docker annotator: %s, scale: %d",getDockerImageName(),getScale(1)); - } - else if(getDriver().equals(DUUISwarmDriver.class.getCanonicalName())) { - return String.format("Swarm annotator: %s, scale: %d",getDockerImageName(),getScale(1)); - } - else if(getDriver().equals(DUUIRemoteDriver.class.getCanonicalName())) { + if (getDriver().equals(DUUIUIMADriver.class.getCanonicalName())) { + return String.format("UIMA annotator: %s, scale: %d", asUIMADriverComponent().getAnnotatorName(), getScale(1)); + } else if (getDriver().equals(DUUIDockerDriver.class.getCanonicalName())) { + return String.format("Docker annotator: %s, scale: %d", getDockerImageName(), getScale(1)); + } else if (getDriver().equals(DUUISwarmDriver.class.getCanonicalName())) { + return String.format("Swarm annotator: %s, scale: %d", getDockerImageName(), getScale(1)); + } else if (getDriver().equals(DUUIRemoteDriver.class.getCanonicalName())) { String urls = "["; - for(String x : getUrl()) { - urls+=x; - urls+=","; + for (String x : getUrl()) { + urls += x; + urls += ","; } - urls = urls.substring(0,urls.length()-1)+"]"; - return String.format("Remote annotator: %s, scale: %d",urls,getScale(1)); - } - else { + urls = urls.substring(0, urls.length() - 1) + "]"; + return String.format("Remote annotator: %s, scale: %d", urls, getScale(1)); + } else { return "Unkown annotator driver!"; } } @@ -683,7 +698,6 @@ public long getTimeout() { return Long.valueOf(_parameters.getOrDefault(timeout, "60")); } - //1 public DUUIPipelineComponent withSlurmJobName(String jobName) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -696,32 +710,20 @@ public DUUIPipelineComponent withSlurmJobName(String jobName) { _options.put(componentName, jobName); return this; } - //2 - public DUUIPipelineComponent withSlurmImagePort(String port) { + + public DUUIPipelineComponent withSlurmHostPort(String port) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } if (port == null) { - _options.remove(slurmImagePort); + _options.remove(slurmHostPort); return this; } - _options.put(slurmImagePort, port); + _options.put(slurmHostPort, port); return this; } -//3 - // public DUUIPipelineComponent withSlurmHostPort(String port) { -// if (_finalizedEncoded != null) { -// throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); -// } -// if (port == null) { -// _options.remove(slurmHostPort); -// return this; -// } -// _options.put(slurmHostPort, port); -// return this; -// } -//4 + public DUUIPipelineComponent withSlurmRuntime(String time) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -734,32 +736,19 @@ public DUUIPipelineComponent withSlurmRuntime(String time) { return this; } - //5 - public DUUIPipelineComponent withSlurmCpus(String num) { - if (_finalizedEncoded != null) { - throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); - } - if (num == null) { - _options.remove(slurmCpus); - return this; - } - _options.put(slurmCpus, num); - return this; - } - //6 - public DUUIPipelineComponent withSlurmMemory(String numG) { + public DUUIPipelineComponent withSlurmMemory(String numMB) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if (numG == null) { + if (numMB == null) { _options.remove(slurmMemory); return this; } - _options.put(slurmMemory, numG); + _options.put(slurmMemory, numMB); return this; } - //7 + public DUUIPipelineComponent withSlurmOutPutLocation(String loc) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -771,7 +760,7 @@ public DUUIPipelineComponent withSlurmOutPutLocation(String loc) { _options.put(slurmOutPutLocation, loc); return this; } - //8 + public DUUIPipelineComponent withSlurmErrorLocation(String loc) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -783,20 +772,19 @@ public DUUIPipelineComponent withSlurmErrorLocation(String loc) { _options.put(slurmErrorLocation, loc); return this; } - //9 + public DUUIPipelineComponent withSlurmSaveIn(String saveTo) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } if (saveTo == null) { - _options.remove(slurmSIFDiskLocation); + _options.remove(slurmSIFLocation); return this; } - _options.put(slurmSIFDiskLocation, saveTo); + _options.put(slurmSIFLocation, saveTo); return this; } - //10 public DUUIPipelineComponent withSlurmGPU(String num) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -808,7 +796,19 @@ public DUUIPipelineComponent withSlurmGPU(String num) { _options.put(slurmGPU, num); return this; } - //11 + + public DUUIPipelineComponent withSlurmCPUs(String num) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (num == null) { + _options.remove(slurmCpus); + return this; + } + _options.put(slurmCpus, num); + return this; + } + public DUUIPipelineComponent withSlurmSIFName(String sifName) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); @@ -821,41 +821,79 @@ public DUUIPipelineComponent withSlurmSIFName(String sifName) { return this; } - //12 - public DUUIPipelineComponent withSlurmEntryLocation(String loc) { + + + public DUUIPipelineComponent withSlurmUvicorn(String loc) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } if (loc == null) { - _options.remove(slurmEntryLocation); + _options.remove(slurmUvicorn); return this; } - _options.put(slurmEntryLocation, loc); + _options.put(slurmUvicorn, loc); return this; } - public DUUIPipelineComponent withSlurmUvicorn(String loc) { + + + public DUUIPipelineComponent withSlurmScript(String script) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if (loc == null) { - _options.remove(slurmUvicorn); + if (script == null) { + _options.remove(slurmScript); return this; } - _options.put(slurmUvicorn, loc); + _options.put(slurmScript, script); return this; } - public DUUIPipelineComponent withSlurmRepoLocation(String loc) { + public DUUIPipelineComponent withSlurmPartition(String pname) { if (_finalizedEncoded != null) { throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); } - if (loc == null) { - _options.remove(slurmSIFRepoLocation); + if (pname == null) { + _options.remove(slurmPartition); + return this; + } + _options.put(slurmPartition, pname); + return this; + } + + public DUUIPipelineComponent withSlurmNodelist(String nodelist) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (nodelist == null) { + _options.remove(slurmNodelist); + return this; + } + _options.put(slurmNodelist, nodelist); + return this; + } + + public DUUIPipelineComponent withSlurmWorkDir(String wd) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (wd == null) { + _options.remove(slurmWorkDir); return this; } - _options.put(slurmSIFRepoLocation, loc); + _options.put(slurmWorkDir, wd); return this; + } + public DUUIPipelineComponent withSlurmInnerPort(String port) { + if (_finalizedEncoded != null) { + throw new RuntimeException("DUUIPipelineComponent has already been finalized, it is immutable now!"); + } + if (port == null) { + _options.remove(slurmInnerPort); + return this; + } + _options.put(slurmInnerPort, port); + return this; } @@ -864,44 +902,35 @@ public String getSlurmSIFImageName() { } public String getSlurmGPU() { - return _options.get(slurmGPU); + return _options.get(slurmGPU); } - public String getSlurmSIFDiskLocation() { - return _options.get(slurmSIFDiskLocation); + public String getSlurmSIFLocation() { + return _options.get(slurmSIFLocation); } public String getSlurmOutPutLocation() { - return _options.get(slurmOutPutLocation); + return _options.get(slurmOutPutLocation); } public String getSlurmMem() { - return _options.get(slurmMemory); - } - - public String getSlurmCpus() { - return _options.get(slurmCpus); + return _options.get(slurmMemory); } public String getSlurmRuntime() { - return _options.get(slurmRuntime); + return _options.get(slurmRuntime); } - public String getSlurmImagePort() { - return _options.get(slurmImagePort); + public String getSlurmHostPort() { + return _options.get(slurmHostPort); } - // public String getSlurmHostPort() { -// return _options.get(slurmHostPort); -// } + public String getSlurmJobName() { - return _options.get(slurmJobName); - } - public String getSlurmErrorLocation() { - return _options.get(slurmErrorLocation); + return _options.get(slurmJobName); } - public String getSlurmEntryLocation() { - return _options.get(slurmEntryLocation); + public String getSlurmErrorLocation() { + return _options.get(slurmErrorLocation); } public Boolean getSlurmRunAfterExit(Boolean defaultValue) { @@ -910,14 +939,32 @@ public Boolean getSlurmRunAfterExit(Boolean defaultValue) { return Boolean.parseBoolean(result); } - public String getSlurmSIFRepoLocation() { - return _options.get(slurmSIFRepoLocation); + public String getSlurmUvicorn() { + return _options.get(slurmUvicorn); + } + public String getSlurmScript() { + return _options.get(slurmScript); } - public String getSlurmUvicorn() { - return _options.get(slurmUvicorn); + public String getSlurmPartition() { + return _options.get(slurmPartition); } + public String getSlurmNodelist() { + return _options.get(slurmNodelist); + } + + public String getSlurmWorkDir() { + return _options.get(slurmWorkDir); + } + + public String getSlurmInnerPort() { + return _options.get(slurmInnerPort); + } + + public String getSlurmCpus() { + return _options.get(slurmCpus); + } } diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java index 8d48e53a..a652f15a 100644 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/IDUUIDriverInterface.java @@ -98,11 +98,11 @@ public interface IDUUIDriverInterface { * @param uuid * @return */ - public boolean destroy(String uuid); + public boolean destroy(String uuid) throws IOException, InterruptedException; /** * Shutting down the driver */ - public void shutdown(); + public void shutdown() throws IOException, InterruptedException; } diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java index 86f0216b..223fe552 100755 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmDriver.java @@ -1,5 +1,6 @@ package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; +import io.vertx.core.cli.annotations.Hidden; import org.apache.commons.compress.compressors.CompressorException; import org.apache.uima.UIMAException; import org.apache.uima.cas.CASException; @@ -16,6 +17,7 @@ import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.DUUIWebsocketAlt; import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.IDUUIConnectionHandler; import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.*; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest; import org.texttechnologylab.DockerUnifiedUIMAInterface.exception.PipelineComponentException; import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaCommunicationLayer; import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; @@ -23,10 +25,8 @@ import org.texttechnologylab.DockerUnifiedUIMAInterface.segmentation.DUUISegmentationStrategy; import org.xml.sax.SAXException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringWriter; +import javax.sound.sampled.Port; +import java.io.*; import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; @@ -48,11 +48,10 @@ import java.util.logging.Logger; import static java.lang.String.format; -import static org.bouncycastle.oer.its.ieee1609dot2.SignerIdentifier.digest; -import static org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.PortManager.waitUntilReachable; + public class DUUISlurmDriver implements IDUUIDriverInterface { - // Man kann den Docker-Client dazu verwenden, ein Image herunterzuladen (per Pull). + private SlurmRest _rest; private DUUIDockerInterface _docker_interface; private DUUILuaContext _luaContext; private DUUISlurmInterface _interface; @@ -63,9 +62,10 @@ public class DUUISlurmDriver implements IDUUIDriverInterface { private IDUUIConnectionHandler _wsclient; private final static Logger LOGGER = Logger.getLogger(DUUIComposer.class.getName()); - //A - public DUUISlurmDriver() throws IOException, UIMAException, SAXException { - _interface = new DUUISlurmInterface(); + + public DUUISlurmDriver(SlurmRest rest) throws IOException, UIMAException, SAXException { + _rest = rest; + _interface = new DUUISlurmInterface(rest); _client = HttpClient.newHttpClient(); JCas _basic = JCasFactory.createJCas(); @@ -80,14 +80,12 @@ public DUUISlurmDriver() throws IOException, UIMAException, SAXException { _luaContext = null; } - //B - /** * Creation of the communication layer based on the Driver * * @param url localhost * @param jc jcas - * @param http_timeout_s timeout with fastapi + * @param http_timeout_s timeout with fastapi * @param client communication wtih fastapi * @param context lua * @param skipVerification true @@ -189,21 +187,18 @@ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, i } ByteArrayOutputStream stream = new ByteArrayOutputStream(); - // cas seri try { - //TODO: Make this accept options to better check the instantiation! + layer.serialize(jc, stream, null, "_InitialView"); } catch (Exception e) { e.printStackTrace(); throw new Exception(format("The serialization step of the communication layer fails for implementing class %s", layer.getClass().getCanonicalName())); } -// http req HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS)) .version(HttpClient.Version.HTTP_1_1) .POST(HttpRequest.BodyPublishers.ofByteArray(stream.toByteArray())) .build(); - // resp HttpResponse resp = client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join(); if (resp.statusCode() == 200) { @@ -214,7 +209,6 @@ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, i System.err.printf("Caught exception printing response %s\n", new String(resp.body(), StandardCharsets.UTF_8)); throw e; } - // use for shallow copy return layer; } else { throw new Exception(format("The container returned response with code != 200\nResponse %s", resp.body().toString())); @@ -222,29 +216,29 @@ public static IDUUICommunicationLayer responsiveAfterTime(String url, JCas jc, i } - //C public void setLuaContext(DUUILuaContext luaContext) { _luaContext = luaContext; } - //------------------------------------------------------------------------------------------------------------- - //D no use -//------------------------------------------------------------------------------------------------------------- - //E + public boolean canAccept(DUUIPipelineComponent comp) { - return comp.getSlurmSIFImageName() != null; + + return comp.getSlurmScript()!= null || !comp.getSlurmRuntime().isEmpty(); + + } -//------------------------------------------------------------------------------------------------------------- - //F + /** - * Instantiate the component - * + * The original code has changed a lot, so please check the interlinear comments. * @param component * @param jc * @param skipVerification + * @param shutdown * @return - * @throws Exception + * @throws InterruptedException + * @throws URISyntaxException + * @throws IOException */ public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skipVerification, AtomicBoolean shutdown) throws InterruptedException, URISyntaxException, IOException { String uuid = UUID.randomUUID().toString(); @@ -255,34 +249,7 @@ public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skip DUUISlurmDriver.InstantiatedComponent comp = new DUUISlurmDriver.InstantiatedComponent(component); - // Inverted if check because images will never be pulled if !comp.getImageFetching() is checked. - - - -// if (!(component.getSlurmSIFRepoLocation() == null)) { -// // mean local no sif file -// //_docker_interface.pullImage(component.getDockerImageName()); -// sif iamge name in dockerimagename -// SlurmUtils.pullSifImagefromRemoteDockerRepo(component.getSlurmSIFImageName(), component.getDockerImageName()); -// } -// // -// else - - // etwas neues - // - if (!(component.getSlurmSIFDiskLocation() == null)) { - //SlurmUtils.pullSifImagefromLocalDockerRepo(component.getSlurmSIFImageName(),component.getSlurmSIFImageName()); - - - System.out.println("sif file existed"); - } else { - throw new RuntimeException("SIF File conflict detected, either local disk location or remote repo url "); - } System.out.printf("[SlurmDriver] Assigned new pipeline component unique id %s\n", uuid); -// String digest = _interface.getDigestFromImage(comp.getImageName()); - - //comp.getPipelineComponent().__internalPinDockerImage(comp.getImageName(), digest); - //System.out.printf("[DockerLocalDriver] Transformed image %s to pinnable image name %s\n", comp.getImageName(), comp.getPipelineComponent().getDockerImageName()); _active_components.put(uuid, comp); @@ -290,80 +257,97 @@ public String instantiate(DUUIPipelineComponent component, JCas jc, boolean skip if (shutdown.get()) { return null; } -// - int hostPort = PortManager.acquire(); - System.out.println("prepare to reserve port " + hostPort); - // dient als placeholder - try (PortManager.PortReservation r = new PortManager.PortReservation(hostPort)) { - // + // If only one component is started, the user can specify a port on their own using withSlurmHostPort() + // Otherwise it will be assigned by portmanager + int hostPort = -1; + String slurmHostPort = component.getSlurmHostPort(); + if (slurmHostPort == null || slurmHostPort.isEmpty()) { + System.out.println("[SlurmDriver] Assigning a free port follows the FIFO-ALGORITHM"); + hostPort = PortManager.acquire(); + } else { + System.out.println("[SlurmDriver] Assigning a port of your choice."); + hostPort = Integer.parseInt(slurmHostPort); + } + //Users can generate scripts from the methods in SlurmUtil by entering a series of parameters, + // or they can just enter a script in json format. + System.out.println("[SlurmDriver] prepare to reserve port: " + hostPort); + String job_id; + if (component.getSlurmScript() != null) { + String jobID = _interface.run_json(component, hostPort); + job_id = jobID; + System.out.println("[SlurmDriver] Submit successfully, Job ID: " + jobID); + + + } else { // - // placeholder muss vor dem Lauf eines Komponent freilassen werden, sonst entsteht ein Konflikt - r.close(); String jobID = _interface.run(component, hostPort); - String port = _interface.extractPort(jobID); // - if (!port.equals(Integer.toString(hostPort))) { - throw new RuntimeException("port mismatch"); - } - + job_id = jobID; + System.out.println("[SlurmDriver] submit " + component.getSlurmJobName() + " successfully, Job ID: " + jobID); + } + // Since submitting tasks in slurm is asynchronous, + // the java code must be blocked to continue running, which prevents http error + boolean ok = PortManager.waitUntilHttpReachable( + "http://localhost:" + hostPort + "/v1/communication_layer", + 200, + null, + Duration.ofMinutes(1), + Duration.ofSeconds(2) + ); + + if (!ok) throw new IllegalStateException("timeout waiting for http "); + + // DUUISlurmInterface maintains a thread-safe hashmap, see DUUISlurmInterface.class for details. + String portFromMap = _interface.extractPort(job_id); + if (!portFromMap.equals(Integer.toString(hostPort))) { + throw new RuntimeException("port mismatch"); + } - //wait test if ok pass - waitUntilReachable("127.0.0.1", hostPort, Duration.ofMinutes(1)); + try { + final int iCopy = i; + final String uuidCopy = uuid; + IDUUICommunicationLayer layer = responsiveAfterTime("http://127.0.0.1" + ":" + hostPort, jc, _http_timeout_s, _client, _luaContext, skipVerification); + if (component.getScale()==null) + { + System.out.println("[SlurmDriver] " + component.getSlurmSIFImageName() + "[" + Integer.toString(iCopy + 1) + "/" + "1" + "]" + "is online and seems to understand DUUI V1 format!"); - try { - final int iCopy = i; - final String uuidCopy = uuid; - IDUUICommunicationLayer layer = responsiveAfterTime("http://127.0.0.1" + ":" + port, jc, _http_timeout_s, _client, _luaContext, skipVerification); + } - if (comp.isWebsocket()) { - String url = "ws://127.0.0.1:" + port; - _wsclient = new DUUIWebsocketAlt( - url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS_WEBSOCKET, comp.getWebsocketElements()); - } else { - _wsclient = null; - } - /** - * @see - * @edited - * Dawit Terefe - * - * Saves websocket client in ComponentInstance for - * retrieval in process_handler-function. - */ - comp.addInstance(new DUUISlurmDriver.ComponentInstance(jobID, hostPort, layer, _wsclient)); - } catch (Exception e) { - //_interface.stop_container(containerid); - //throw e; + System.out.println("[SlurmDriver] " + component.getSlurmSIFImageName() + "[" + Integer.toString(iCopy + 1) + "/" + component.getScale() + "]" + "is online and seems to understand DUUI V1 format!"); + if (comp.isWebsocket()) { + String url = "ws://127.0.0.1:" + hostPort; + _wsclient = new DUUIWebsocketAlt( + url + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS_WEBSOCKET, comp.getWebsocketElements()); + } else { + _wsclient = null; } + + comp.addInstance(new DUUISlurmDriver.ComponentInstance(job_id, hostPort, layer, _wsclient)); + } catch (Exception e) { + } + } return shutdown.get() ? null : uuid; } + + /** + * Show the maximum parallelism + * + * @param uuid + */ @Override public void printConcurrencyGraph(String uuid) { - + DUUISlurmDriver.InstantiatedComponent component = _active_components.get(uuid); + if (component == null) { + throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); + } + System.out.printf("[DockerLocalDriver][%s]: Maximum concurrency %d\n", uuid, component.getInstances().size()); } -////G no use -// /** -// * Show the maximum parallelism -// * -// * @param uuid -// */ -// public void printConcurrencyGraph(String uuid) { -// DUUISlurmDriver.InstantiatedComponent component = _active_components.get(uuid); -// if (component == null) { -// throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); -// } -// System.out.printf("[DockerLocalDriver][%s]: Maximum concurrency %d\n", uuid, component.getInstances().size()); -// } - -//----------------------------------------------------------------------------------------------------------------------- -//H uuid 1--------------------n same type component - /** * Return the TypeSystem used by the given Component * @@ -383,36 +367,25 @@ public TypeSystemDescription get_typesystem(String uuid) throws InterruptedExcep return IDUUIInstantiatedPipelineComponent.getTypesystem(uuid, comp); } + + /** + * init reader component + * + * @param uuid + * @param filePath + * @return + */ @Override - public int initReaderComponent(String uuid, Path filePath) throws Exception { - return 0; + public int initReaderComponent(String uuid, Path filePath) { + InstantiatedComponent comp = _active_components.get(uuid); + if (comp == null) { + throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); + } + return IDUUIInstantiatedPipelineReaderComponent.initComponent(comp, filePath); } - - - /// /I what is this? -// /** -// * init reader component -// * @param uuid -// * @param filePath -// * @return -// */ -// @Override -// public int initReaderComponent(String uuid, Path filePath) { -// InstantiatedComponent comp = _active_components.get(uuid); -// if (comp == null) { -// throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); -// } -// return IDUUIInstantiatedPipelineReaderComponent.initComponent(comp, filePath); -// } //------------------------------------------------------------------------------------------------------------- -//J over write service logic - - - //K no use -//L close no use - //M public static class ComponentInstance implements IDUUIUrlAccessible { private String _job_ID; private int _hostPort; @@ -446,9 +419,6 @@ public int get_hostPort() { return _hostPort; } -// public int get_fastAPIPort() { -// return _fastAPIPort; -// } public IDUUIConnectionHandler get_handler() { return _handler; @@ -471,9 +441,6 @@ public IDUUIConnectionHandler getHandler() { } } - //------------------------------------------------------------------------------------ -//J - /** * Execute a component in the driver * @@ -496,12 +463,7 @@ public void run(String uuid, JCas aCas, DUUIPipelineDocumentPerformance perf, DU } } - @Override - public boolean destroy(String uuid) { - return false; - } - //N static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent { private String _image_name; private ConcurrentLinkedQueue _instances; @@ -511,16 +473,12 @@ static class InstantiatedComponent implements IDUUIInstantiatedPipelineComponent private boolean _withImageFetching; private boolean _websocket; private int _ws_elements; - - //private String _reg_password; - //private String _reg_username; private String _uniqueComponentKey; private Map _parameters; private String _sourceView; private String _targetView; private DUUIPipelineComponent _component; - // 这里？？ public Triplet getComponent() { long mutexStart = System.nanoTime(); DUUISlurmDriver.ComponentInstance inst = _instances.poll(); @@ -531,7 +489,6 @@ public Triplet getComponent() { return Triplet.with(inst, mutexStart, mutexEnd); } - //-------------------------------------------------------------------------------- public void addComponent(IDUUIUrlAccessible access) { _instances.add((DUUISlurmDriver.ComponentInstance) access); } @@ -555,9 +512,6 @@ public void addComponent(IDUUIUrlAccessible access) { _keep_runnging_after_exit = comp.getSlurmRunAfterExit(false); - // _reg_password = comp.getDockerAuthPassword(); - // _reg_username = comp.getDockerAuthUsername(); - _websocket = comp.isWebsocket(); _ws_elements = comp.getWebsocketElements(); } @@ -570,14 +524,6 @@ public String getUniqueComponentKey() { return _uniqueComponentKey; } -// public String getPassword() { -// return _reg_password; -// } - - // public String getUsername() { - // return _reg_username; - // } - public boolean getImageFetching() { return _withImageFetching; } @@ -628,188 +574,68 @@ public int getWebsocketElements() { } - //O public static class Component { private DUUIPipelineComponent _component; - public DUUISlurmDriver.Component withParameter(String key, String value) { - _component.withParameter(key, value); - return this; - } - - public DUUISlurmDriver.Component withView(String viewName) { - _component.withView(viewName); - return this; - } - - public DUUISlurmDriver.Component withSourceView(String viewName) { - _component.withSourceView(viewName); - return this; - } - - public DUUISlurmDriver.Component withTargetView(String viewName) { - _component.withTargetView(viewName); - return this; - } - - public Component(String target) throws URISyntaxException, IOException { - _component = new DUUIPipelineComponent(); - _component.withDockerImageName(target); - } public Component(DUUIPipelineComponent pComponent) throws URISyntaxException, IOException { _component = pComponent; _component.withDriver(DUUISlurmDriver.class); } - public DUUISlurmDriver.Component withDescription(String description) { - _component.withDescription(description); - return this; - } - - public DUUISlurmDriver.Component withScale(int scale) { - _component.withScale(scale); - return this; - } - -// public DUUISlurmDriver.Component withRegistryAuth(String username, String password) { -// _component.withDockerAuth(username, password); -// return this; -// } - -// public DUUISlurmDriver.Component withImageFetching() { -// return withImageFetching(true); -// } -// -// public DUUISlurmDriver.Component withImageFetching(boolean imageFetching) { -// _component.withDockerImageFetching(imageFetching); -// return this; -// } - -// public DUUISlurmDriver.Component withGPU(boolean gpu) { -// _component.withDockerGPU(gpu); -// return this; -// } - -// public DUUISlurmDriver.Component withRunningAfterDestroy(boolean run) { -// _component.withDockerRunAfterExit(run); -// return this; -// } - -// public DUUISlurmDriver.Component withWebsocket(boolean b) { -// _component.withWebsocket(b); -// return this; -// } -// -// public DUUISlurmDriver.Component withWebsocket(boolean b, int elements) { -// _component.withWebsocket(b, elements); -// return this; -// } - - public DUUISlurmDriver.Component withSegmentationStrategy(DUUISegmentationStrategy strategy) { - _component.withSegmentationStrategy(strategy); - return this; - } - - public DUUISlurmDriver.Component withSegmentationStrategy(Class strategyClass) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { - _component.withSegmentationStrategy(strategyClass.getDeclaredConstructor().newInstance()); - return this; - } - public DUUIPipelineComponent build() { return _component; } -// public DUUISlurmDriver.Component withName(String name) { -// _component.withName(name); -// return this; -// } - - - //1 - public Component withSlurmJobName(String jobName) { - _component.withSlurmJobName(jobName); - return this; - } - - //2 - public Component withSlurmImagePort(String port) { - _component.withSlurmImagePort(port); - return this; - } - - - //3 -// public Component withSlurmHostPort(String port) { -// _component.withSlurmHostPort(port); -// return this; -// } - - //4 - public Component withSlurmRuntime(String time) { - _component.withSlurmRuntime(time); - return this; - } - - //5 - public Component withSlurmCpus(String num) { - _component.withSlurmCpus(num); - return this; - } - - //6 - public Component withSlurmMemory(String numG) { - _component.withSlurmMemory(numG); - return this; - } - - //7 - public Component withSlurmOutPutLocation(String loc) { - _component.withSlurmOutPutLocation(loc); - return this; - } + } - //8 - public Component withSlurmErrorLocation(String loc) { - _component.withSlurmErrorLocation(loc); - return this; - } + @Override + public void shutdown() throws IOException, InterruptedException{ + String token = _rest.generateRootToken("slurmctld"); + System.out.println("shutdown hook clean all jobs"); + HashMap jobIDPortMap = _interface.get_jobID_PortMap(); + jobIDPortMap.keySet().forEach(jobID -> { + try { + _rest.cancelJob(token, jobID); - //9 - public Component withSlurmSaveIn(String saveTo) { - _component.withSlurmSaveIn(saveTo); - return this; - } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } - //10 - public Component withSlurmGPU(String num) { - _component.withSlurmGPU(num); - return this; - } - //11 - public Component withSlurmSIFName(String sifName) { - _component.withSlurmSIFName(sifName); - return this; + /** + * Because it is a self-maintained port table, + * it allows the port to be released to another component for continued use after the end of each type of component. + * See PortManager.class for details + * @param uuid + * @return + * @throws IOException + * @throws InterruptedException + */ + @Override + public boolean destroy(String uuid) throws IOException, InterruptedException { + String token = _rest.generateRootToken("slurmctld"); + DUUISlurmDriver.InstantiatedComponent comp = _active_components.remove(uuid); + boolean flag = true; + if (comp == null) { + throw new InvalidParameterException("Invalid UUID, this component has not been instantiated by the local Driver"); } + if (!comp.getRunningAfterExit()) { + int counter = 1; + for (DUUISlurmDriver.ComponentInstance inst : comp.getInstances()) { + String jobId = inst.get_job_ID(); + boolean status = _rest.cancelJob(token, jobId); + PortManager.release(inst.get_hostPort()); + _interface.get_jobID_PortMap().remove(jobId); + counter += 1; + flag = flag && status; - //12 - public Component withSlurmEntryLocation(String loc) { - _component.withSlurmEntryLocation(loc); - return this; - + } } - - - } - -//-------------------------------------------------------------------------- - - - @Override - public void shutdown() { - + return flag; } } diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java index 12ae6701..bb6c4813 100755 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/DUUISlurmInterface.java @@ -4,98 +4,93 @@ import com.github.dockerjava.core.DockerClientBuilder; import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; import com.github.dockerjava.transport.DockerHttpClient; +import com.influxdb.client.JSON; +import org.json.JSONObject; import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest; import java.io.IOException; import java.net.URI; import java.time.Duration; import java.util.HashMap; -// funtionen: 1 use docker cli pull image falls nicht vorhanden -// 2 in slurm hat jede Aufgabe eindeutig ein jobid, man kann anhand id ein job starten oder stoppen， -// also eine Verwalttung von Port und id sehr wichtig Hashmap + public class DUUISlurmInterface { - private DockerClient _docker; + private SlurmRest slurmRest; private HashMap _jobID_PortMap = new HashMap<>(); - public DUUISlurmInterface() throws IOException { -// - if (!System.getProperty("os.name").contains("Windows")) { - _docker = DockerClientBuilder.getInstance().build(); - } else { - // Windows - DockerHttpClient http = null; - try { - http = new ApacheDockerHttpClient.Builder() - .connectionTimeout(Duration.ofSeconds(5)) - .responseTimeout(Duration.ofMinutes(10)) - .dockerHost(URI.create("npipe:////./pipe/docker_engine")) - .build(); - } catch (Exception e) { - http = new ApacheDockerHttpClient.Builder() - .connectionTimeout(Duration.ofSeconds(5)) - .responseTimeout(Duration.ofMinutes(10)) - .dockerHost(URI.create("tcp://127.0.0.1:2375")) // if npipe doesn't work. - .build(); - } - _docker = DockerClientBuilder.getInstance() - .withDockerHttpClient(http) - .build(); - } - - + public DUUISlurmInterface(SlurmRest rest) throws IOException { + slurmRest = rest; } - - public boolean checkDependencies() { - return SlurmUtils.checkSlurmInstalled() && SlurmUtils.checkSocatInstalled(); - } - - // Die Funktion run erstellt ein Shell-Skript, das dem Slurm-Jobsystem übergeben und dann ausgeführt wird.“ + /** + * Generate a script based on the parameters given by the user, submit and execute it + * @param comp + * @param hostPort + * @return + * @throws IOException + * @throws InterruptedException + */ public String run(DUUIPipelineComponent comp, int hostPort) throws IOException, InterruptedException { + String hport = Integer.toString(hostPort); String gpu = comp.getSlurmGPU(); String sifName = comp.getSlurmSIFImageName(); String slurmJobName = comp.getSlurmJobName(); - String slurmEntryLocation = comp.getSlurmEntryLocation(); - String slurmErrorLocation = comp.getSlurmErrorLocation(); - String slurmImagePort = comp.getSlurmImagePort(); - String slurmCpus = comp.getSlurmCpus(); String slurmRuntime = comp.getSlurmRuntime(); String slurmMem = comp.getSlurmMem(); - String slurmSaveIn = comp.getSlurmSIFDiskLocation(); - String slurmOutPutLocation = comp.getSlurmOutPutLocation(); + String slurmSaveIn = comp.getSlurmSIFLocation(); String slurmUvicorn = comp.getSlurmUvicorn(); -//Diese Funktion konvertiert einen String in ein Shell Format. - String jobid = SlurmUtils.submitJob( - slurmJobName, - Integer.toString(hostPort), - slurmCpus, gpu, slurmEntryLocation, - slurmErrorLocation, - slurmImagePort, - slurmMem, - slurmRuntime, - slurmOutPutLocation, - sifName, - slurmSaveIn, - slurmUvicorn - ); - - System.out.println("job submitted ready for release, jobid : " + jobid); - // neue Abbildung gespeichert - _jobID_PortMap.put(jobid, Integer.toString(hostPort)); - - return jobid; - } + String slurmNodelist = comp.getSlurmNodelist(); + String slurmPartition = comp.getSlurmPartition(); + String slurmWorkDir = comp.getSlurmWorkDir(); + String slurmCpus = comp.getSlurmCpus(); + JSONObject j = new JSONObject(); + if (Integer.parseInt(gpu)>0){ + JSONObject jsonObject = SlurmUtils.generateJobScript_GPU(slurmJobName, slurmRuntime, slurmWorkDir, hport, slurmSaveIn, slurmUvicorn, slurmMem, gpu, sifName, slurmPartition, slurmNodelist, slurmCpus); + j = jsonObject; + } + else { + j = SlurmUtils.generateJobScript_non_GPU(slurmJobName,slurmRuntime,slurmWorkDir,hport,slurmSaveIn, slurmUvicorn,slurmMem,sifName,slurmPartition,slurmNodelist, slurmCpus); + } + String token = slurmRest.generateRootToken("slurmctld"); + String[] submit = slurmRest.submit(token, j); + //System.out.println(submit[0]); + String id = submit[1]; + // System.out.println("[SlurmDriver] Job submitted , jobid : " + id); + _jobID_PortMap.put(id, Integer.toString(hostPort)); + + return id; + } - public String extractPort(String jobid){ - return _jobID_PortMap.get(jobid); + /** + * submit and execute json script + * @param comp + * @param hostPort + * @return + * @throws IOException + * @throws InterruptedException + */ + public String run_json(DUUIPipelineComponent comp, int hostPort) throws IOException, InterruptedException { + String slurmScript = comp.getSlurmScript(); + JSONObject slurmJson = new JSONObject(slurmScript); + if (slurmRest.checkRESTD()) { + String token = slurmRest.generateRootToken("slurmctld"); + String[] submit = slurmRest.submit(token, slurmJson); + //System.out.println(submit[0]); + _jobID_PortMap.put(submit[1], Integer.toString(hostPort)); + return submit[1]; + } + else { + throw new RuntimeException("slurm rest check failed"); + } } - public DockerClient get_docker() { - return _docker; + + public String extractPort(String jobid) { + return _jobID_PortMap.get(jobid); } public HashMap get_jobID_PortMap() { diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java new file mode 100755 index 00000000..8802d19b --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/PortManager.java @@ -0,0 +1,108 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; + +/** + * he port numbers allowed here should be the same as ports:- xx:xx in docker-compose.yml. + * Due to the algorithmic multiplexing of ports using fifo, + * each component theoretically opens up a maximum of MAX-BASE+1 at the same time. + * If there is not enough hardware to allocate, an error is reported. + */ +public class PortManager { + private static final int BASE = 20000; + private static final int MAX = 20100; + private static final AtomicInteger next = new AtomicInteger(BASE); + private static final ConcurrentLinkedQueue reuse_queue = new ConcurrentLinkedQueue<>(); + + public static int acquire() throws IOException { + Integer fromPool = reuse_queue.poll(); + if (fromPool != null) return fromPool; + + int candidate = next.getAndIncrement(); + if (candidate >= MAX) { + throw new IOException("No free port in pool (" + BASE + "‑" + (MAX) + ')'); + } + return candidate; + } + + public static void release(int port) { + if (port < BASE || port >= MAX) return; + reuse_queue.offer(port); + } + + + public static int remaining() { + int produced = next.get() - BASE; + int total = MAX - BASE; + return total - produced + reuse_queue.size(); + } + + public static boolean isHttpReachable(String url, + int expectedStatus, + Predicate bodyPredicate, + int timeoutMillis) { + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setConnectTimeout(timeoutMillis); + conn.setReadTimeout(timeoutMillis); + conn.setRequestMethod("GET"); + + int status = conn.getResponseCode(); + if (status != expectedStatus) { + return false; + } + + if (bodyPredicate == null) { + return true; + } + + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(conn.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return bodyPredicate.test(sb.toString()); + } + } catch (IOException e) { + return false; + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + + public static boolean waitUntilHttpReachable(String url, + int expectedStatus, + Predicate bodyPredicate, + Duration maxWait, + Duration interval) throws InterruptedException { + Instant deadline = Instant.now().plus(maxWait); + while (Instant.now().isBefore(deadline)) { + if (isHttpReachable(url, expectedStatus, bodyPredicate, + (int) interval.toMillis())) { + return true; + } + Thread.sleep(interval.toMillis()); + } + return false; + } + +} + + diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java deleted file mode 100755 index e2e8f719..00000000 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/Portmanager.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.time.Duration; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -// should use ssh -L later -/* 20000–30000 speiziell reserviert fuer slurm */ -final class PortManager { - private static final int BASE = 20000; - private static final int MAX = 30000; - private static final Set RESERVED = ConcurrentHashMap.newKeySet(); - - /** - * get a free port for placeholder -> make it RESERVED - */ - - static int acquire() throws IOException { - for (int p = BASE; p < MAX; p++) { - if (!RESERVED.add(p)) { - continue; - } // conflict - if (isFree(p)) { - return p; - } // success - RESERVED.remove(p); // bind failed - } - throw new IOException("No free port in pool"); - } - - static void release(int port) { - RESERVED.remove(port); - } - - private static boolean isFree(int port) { - try (ServerSocket s = new ServerSocket()) { - s.setReuseAddress(false); - s.bind(new InetSocketAddress("0.0.0.0", port)); - return true; - } catch (IOException e) { - return false; - } - } - - // ich vergesse immer am ende close()anzurufen, benuztze autocloseable, anweisungen in {} verklammern - static class PortReservation implements AutoCloseable { - private final ServerSocket placeholder; - final int port; - - PortReservation(int port) throws IOException { - this.port = port; - this.placeholder = new ServerSocket(); - this.placeholder.setReuseAddress(false); - this.placeholder.bind(new InetSocketAddress("0.0.0.0", port)); - } - - @Override - public void close() throws IOException { - placeholder.close(); - } - } -// schickt ein paar - - static void waitUntilReachable(String host, int port, Duration timeout) - throws IOException, InterruptedException { - - long deadline = System.nanoTime() + timeout.toNanos(); - long backoff = 250; // ms!!! - while (System.nanoTime() < deadline) { - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress(host, port), 1000); - return; - } catch (IOException ignore) { - } - Thread.sleep(backoff); - backoff = Math.min(backoff * 2, 2000); - } - throw new IOException("Port " + port + " did not open within " + timeout.toSeconds() + " s"); - } - - -} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java deleted file mode 100755 index eecae4d2..00000000 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SinfoData.java +++ /dev/null @@ -1,149 +0,0 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; - -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.annotation.Nulls; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.List; - -/** - * cli insruction sinfo in java console - * für zukunftige DEBUG und log - */ -public class SinfoData { - - public static class SinfoResponse { - public List sinfo = List.of(); - public Meta meta = new Meta(); - public List errors = List.of(); - public List warnings = List.of(); - } - - public static class SinfoRecord { - public int port; - public Nodes nodes = new Nodes(); - public Cpus cpus = new Cpus(); - public Node node = new Node(); - public Gres gres = new Gres(); - public Memory memory = new Memory(); - public Partition partition = new Partition(); - - public String comment = "", cluster = "", extra = "", reservation = ""; - } - - - public static class Node { - public List state = List.of(); - } - - public static class Nodes { - public int total, idle, allocated, other; - public List hostnames = List.of(), addresses = List.of(), nodes = List.of(); - } - - public static class Cpus { - public int total, idle, allocated, other; - } - - public static class Gres { - public String total = "", used = ""; - } - - public static class Memory { - public int minimum, maximum, allocated; - public Free free = new Free(); - - public static class Free { - public NumberObj minimum = new NumberObj(), maximum = new NumberObj(); - - public static class NumberObj { - public boolean set; - public boolean infinite; - public int number; - } - } - } - - public static class Partition { - public String name = ""; - public PartitionState partition = new PartitionState(); - - public static class PartitionState { - public List state = List.of(); - } - } - - public static class Meta { - public Slurm slurm = new Slurm(); - - public static class Slurm { - public String release = ""; - } - } - - - private static SinfoResponse runSinfo() { - StringBuilder json = new StringBuilder(); - try { - Process p = new ProcessBuilder("sinfo", "--json").redirectErrorStream(true).start(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - br.lines().forEach(json::append); - } - p.waitFor(); - } catch (Exception e) { - throw new RuntimeException("sinfo failed", e); - } - - ObjectMapper mapper = new ObjectMapper() - - .setDefaultSetterInfo(JsonSetter.Value.forValueNulls(Nulls.AS_EMPTY)) - - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - try { - return mapper.readValue(json.toString(), SinfoResponse.class); - } catch (Exception e) { - throw new RuntimeException("parse json fail...", e); - } - } - - public static void sinfo() { - SinfoResponse resp = runSinfo(); - - String hdrFmt = "%-10s %10s %6s %10s %10s %-10s %-10s %11s %9s %11s%n"; - String rowFmt = "%-10s %6s %8s %10s %10s %-12s %-10s %9d %8d %10d%n"; - - System.out.printf(hdrFmt, - "Partition", "NodeTotal", "Idle", "CPU_Total", "CPU_Idle", - "State", "GRES", "MemTot(MB)", "MemAlloc", "MemFreeMin"); - System.out.println("--------------------------------------------------------------------------------------------------------"); - - for (SinfoRecord r : resp.sinfo) { - System.out.printf(rowFmt, - r.partition.name, - r.nodes.total, - r.nodes.idle, - r.cpus.total, - r.cpus.idle, - r.node.state, - r.gres.total, - r.memory.maximum, - r.memory.allocated, - r.memory.free.minimum.number); - } - System.out.println("--------------------------------------------------------------------------------------------------------"); - System.out.printf("Slurm release: %s | Errors: %s%n", - resp.meta.slurm.release, - resp.errors); - } - - - - -} - - - diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java deleted file mode 100755 index 0e99e739..00000000 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmConfig.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; - -import java.util.HashMap; -@Deprecated -public class SlurmConfig { -// #!/bin/bash -//#SBATCH --job-name=duui -//#SBATCH --cpus-per-task=2 -// #SBATCH --time=1:00:00 -// -// PORT=9000 -// INNER=9714 -// IMG=/home/jd/again.sif -// -// apptainer exec "$IMG" \ -// sh -c "cd /usr/src/app && uvicorn textimager_duui_spacy:app --host 0.0.0.0 --port $INNER" & -// -// PID=$! -// socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER & -// PID_SOCAT=$! -// -// wait $PID -// kill $PID_SOCAT - - private static String slurmJobName = "slurmJobName"; - private static String slurmImagePort = "slurmPort"; - private static String slurmHostPort = "slurmHostPort"; - private static String slurmRuntime = "slurmRuntime"; - private static String slurmCpus = "slurmCpus"; - private static String slurmMemory = "slurmMemory"; - private static String slurmErrorLocation = "slurmErrorLocation"; - private static String slurmOutPutLocation = "slurmOutPutLocation"; - private static String slurmSIFSaveIn = "slurmSIFSaveIn"; - private static String slurmGPU = "slurmGPU"; - private static String slurmSIFImageName = "slurmSIFImageName"; - private static String slurmEntryLocation = "slurmEntryLocation"; - - - private String slurmJobNameV; - private String slurmImagePortV; - private String slurmHostPortV; - private String slurmRuntimeV; - private String slurmCpusV; - private String slurmMemoryV; - private String slurmErrorLocationV; - private String slurmOutPutLocationV; - private String slurmSIFSaveInV; - private String slurmGPUV; - private String slurmSIFImageNameV; - private String entryLocationV; - private HashMap slurmConfigProp = new HashMap(); - - - public SlurmConfig(String slurmJobNameV, String slurmImagePortV, String slurmHostPortV, String slurmRuntimeV, String slurmCpusV, String slurmMemoryV, String slurmErrorLocationV, String slurmOutPutLocationV, String slurmSIFSaveInV, String slurmGPUV, String slurmSIFImageNameV, String entryLocationV) { - this.slurmJobNameV = slurmJobNameV; - this.slurmImagePortV = slurmImagePortV; - this.slurmHostPortV = slurmHostPortV; - this.slurmRuntimeV = slurmRuntimeV; - this.slurmCpusV = slurmCpusV; - this.slurmMemoryV = slurmMemoryV; - this.slurmErrorLocationV = slurmErrorLocationV; - this.slurmOutPutLocationV = slurmOutPutLocationV; - this.slurmSIFSaveInV = slurmSIFSaveInV; - this.slurmGPUV = slurmGPUV; - this.slurmSIFImageNameV = slurmSIFImageNameV; - this.entryLocationV = entryLocationV; - //---------------------------------------------------------------- - slurmConfigProp.put(slurmJobName, slurmJobNameV); - slurmConfigProp.put(slurmImagePort, slurmImagePortV); - slurmConfigProp.put(slurmHostPort, slurmHostPortV); - slurmConfigProp.put(slurmRuntime, slurmRuntimeV); - slurmConfigProp.put(slurmCpus, slurmCpusV); - slurmConfigProp.put(slurmMemory, slurmMemoryV); - slurmConfigProp.put(slurmErrorLocation, slurmErrorLocationV); - slurmConfigProp.put(slurmOutPutLocation, slurmOutPutLocationV); - slurmConfigProp.put(slurmSIFSaveIn, slurmSIFSaveInV); - slurmConfigProp.put(slurmGPU, slurmGPUV); - slurmConfigProp.put(slurmSIFImageName, entryLocationV); - slurmConfigProp.put(slurmEntryLocation, entryLocationV); - - } - - -} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java deleted file mode 100755 index 1781faa1..00000000 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmProcess.java +++ /dev/null @@ -1,274 +0,0 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; - -import org.apache.commons.compress.compressors.CompressorException; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.uima.cas.CASException; -import org.apache.uima.jcas.JCas; -import org.javatuples.Triplet; -import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.IDUUICommunicationLayer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.DUUIWebsocketAlt; -import org.texttechnologylab.DockerUnifiedUIMAInterface.connection.IDUUIConnectionHandler; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.IDUUIInstantiatedPipelineComponent; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.IDUUIUrlAccessible; -import org.texttechnologylab.DockerUnifiedUIMAInterface.pipeline_storage.DUUIPipelineDocumentPerformance; -import org.texttechnologylab.duui.ReproducibleAnnotation; -import org.xml.sax.SAXException; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InvalidObjectException; -import java.net.ProxySelector; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.List; - -public class SlurmProcess { - public static HttpClient _client = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .followRedirects(HttpClient.Redirect.ALWAYS) - .proxy(ProxySelector.getDefault()) - .connectTimeout(Duration.ofSeconds(1000)).build(); - - /** - * Calling the DUUI component - * @param jc - * @param comp - * @param perf - * @throws CompressorException - * @throws IOException - * @throws SAXException - * @throws CASException - */ - public static void process(JCas jc, IDUUIInstantiatedPipelineComponent comp, DUUIPipelineDocumentPerformance perf) throws CompressorException, IOException, SAXException, CASException { - - - - Triplet queue = comp.getComponent(); -// get lua script - IDUUICommunicationLayer layer = queue.getValue0().getCommunicationLayer(); - long serializeStart = System.nanoTime(); - - ByteArrayOutputStream out = new ByteArrayOutputStream(1024*1024); - - DUUIPipelineComponent pipelineComponent = comp.getPipelineComponent(); - String viewName = pipelineComponent.getViewName(); - JCas viewJc; - if(viewName == null) { - viewJc = jc; - } - else { - try { - viewJc = jc.getView(viewName); - } - catch(CASException e) { - if(pipelineComponent.getCreateViewFromInitialView()) { - viewJc = jc.createView(viewName); - viewJc.setDocumentText(jc.getDocumentText()); - viewJc.setDocumentLanguage(jc.getDocumentLanguage()); - } - else { - throw e; - } - } - } - - layer.serialize(viewJc,out,comp.getParameters(), comp.getSourceView()); - // lua serialize call() - - byte[] ok = out.toByteArray(); - long sizeArray = ok.length; - long serializeEnd = System.nanoTime(); - - long annotatorStart = serializeEnd; - int tries = 0; - HttpResponse resp = null; - while (tries < 3) { - tries++; - try { - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(queue.getValue0().generateURL() + DUUIComposer.V1_COMPONENT_ENDPOINT_PROCESS)) - .timeout(Duration.ofSeconds(comp.getPipelineComponent().getTimeout())) - .POST(HttpRequest.BodyPublishers.ofByteArray(ok)) - .version(HttpClient.Version.HTTP_1_1) - .build(); - resp = _client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray()).join(); - break; - } - catch(Exception e) { - e.printStackTrace(); - //System.out.printf("Cannot reach endpoint trying again %d/%d...\n",tries+1,10); - } - } - if(resp==null) { - throw new IOException("Could not reach endpoint after 3 tries!"); - } - - - if (resp.statusCode() == 200) { - ByteArrayInputStream st = new ByteArrayInputStream(resp.body()); - long annotatorEnd = System.nanoTime(); - long deserializeStart = annotatorEnd; - - try { - layer.deserialize(viewJc, st, comp.getTargetView()); - } - catch(Exception e) { - System.err.printf("Caught exception printing response %s\n",new String(resp.body(), StandardCharsets.UTF_8)); - - // TODO better error handleing flow? - comp.addComponent(queue.getValue0()); - - // TODO handle error docs for db here too? - - throw e; - } - long deserializeEnd = System.nanoTime(); - - ReproducibleAnnotation ann = new ReproducibleAnnotation(jc); - ann.setDescription(comp.getPipelineComponent().getFinalizedRepresentation()); - ann.setCompression(DUUIPipelineComponent.compressionMethod); - ann.setTimestamp(System.nanoTime()); - ann.setPipelineName(perf.getRunKey()); - ann.addToIndexes(); - perf.addData(serializeEnd-serializeStart,deserializeEnd-deserializeStart,annotatorEnd-annotatorStart,queue.getValue2()-queue.getValue1(),deserializeEnd-queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, null); - - comp.addComponent(queue.getValue0()); - } else { - comp.addComponent(queue.getValue0()); - ByteArrayInputStream st = new ByteArrayInputStream(resp.body()); - String responseBody = new String(st.readAllBytes(), StandardCharsets.UTF_8); - st.close(); - - // track "performance" of error documents if not explicitly disabled - if (perf.shouldTrackErrorDocs()) { - long annotatorEnd = System.nanoTime(); - long deserializeStart = annotatorEnd; - long deserializeEnd = System.nanoTime(); - - String error = "Expected response 200, got " + resp.statusCode() + ": " + responseBody; - - perf.addData(serializeEnd - serializeStart, deserializeEnd - deserializeStart, annotatorEnd - annotatorStart, queue.getValue2() - queue.getValue1(), deserializeEnd - queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, error); - } - - if (!pipelineComponent.getIgnoringHTTP200Error()) { - throw new InvalidObjectException(String.format("Expected response 200, got %d: %s", resp.statusCode(), responseBody)); - } else { - System.err.println(String.format("Expected response 200, got %d: %s", resp.statusCode(), responseBody)); - } - } - } - - /** - * The process merchant describes the use of the component as a web socket - * @param jc - * @param comp - * @param perf - * @throws CompressorException - * @throws IOException - * @throws SAXException - * @throws CASException - * @throws InterruptedException - */ - public static void process_handler(JCas jc, - IDUUIInstantiatedPipelineComponent comp, - DUUIPipelineDocumentPerformance perf) throws CompressorException, IOException, SAXException, CASException, InterruptedException { - Triplet queue = comp.getComponent(); - - IDUUICommunicationLayer layer = queue.getValue0().getCommunicationLayer(); - long serializeStart = System.nanoTime(); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - - DUUIPipelineComponent pipelineComponent = comp.getPipelineComponent(); - - String viewName = pipelineComponent.getViewName(); - JCas viewJc; - if(viewName == null) { - viewJc = jc; - } - else { - try { - viewJc = jc.getView(viewName); - } - catch(CASException e) { - if(pipelineComponent.getCreateViewFromInitialView()) { - viewJc = jc.createView(viewName); - viewJc.setDocumentText(jc.getDocumentText()); - viewJc.setDocumentLanguage(jc.getDocumentLanguage()); - } - else { - throw e; - } - } - } - // lua serialize call() - layer.serialize(viewJc,out,comp.getParameters(), comp.getSourceView()); - - // ok is the message. - byte[] ok = out.toByteArray(); - long sizeArray = ok.length; - long serializeEnd = System.nanoTime(); - - long annotatorStart = serializeEnd; - - /** - * @edited Givara Ebo, Dawit Terefe - * - * Retrieve websocket-client from IDUUIUrlAccessible (ComponentInstance). - * - */ - IDUUIUrlAccessible accessible = queue.getValue0(); - IDUUIConnectionHandler handler = accessible.getHandler(); - - if (handler.getClass() == DUUIWebsocketAlt.class){ - String error = null; - - JCas finalViewJc = viewJc; - - List results = handler.send(ok); - - long annotatorEnd = System.nanoTime(); - long deserializeStart = annotatorEnd; - - ByteArrayInputStream result = null; - try { - /*** - * @edited - * Givara Ebo, Dawit Terefe - * - * Merging results before deserializing. - */ - result = layer.merge(results); - layer.deserialize(finalViewJc, result, comp.getTargetView()); - } - catch(Exception e) { - e.printStackTrace(); - System.err.printf("Caught exception printing response %s\n",new String(result.readAllBytes(), StandardCharsets.UTF_8)); - - // TODO more error handling needed? - error = ExceptionUtils.getStackTrace(e); - } - - long deserializeEnd = System.nanoTime(); - - comp.addComponent(accessible); - - ReproducibleAnnotation ann = new ReproducibleAnnotation(jc); - ann.setDescription(comp.getPipelineComponent().getFinalizedRepresentation()); - ann.setCompression(DUUIPipelineComponent.compressionMethod); - ann.setTimestamp(System.nanoTime()); - ann.setPipelineName(perf.getRunKey()); - ann.addToIndexes(); - perf.addData(serializeEnd-serializeStart,deserializeEnd-deserializeStart,annotatorEnd-annotatorStart,queue.getValue2()-queue.getValue1(),deserializeEnd-queue.getValue1(), String.valueOf(comp.getPipelineComponent().getFinalizedRepresentationHash()), sizeArray, jc, error); - comp.addComponent(accessible); - } - } - -} diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java index fea80fd2..9dd3ba4b 100755 --- a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/SlurmUtils.java @@ -1,6 +1,9 @@ package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import io.swagger.util.Json; +import org.json.JSONArray; +import org.json.JSONObject; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest; import java.io.BufferedReader; import java.io.IOException; @@ -12,39 +15,19 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Map; public class SlurmUtils { -// ein paar cli command in java umschreiben/kaspseln - public static boolean isMasterNode(){ - String s = whoIsMaster().split("=", 2)[1]; - return s.equals(whoAmI());} + public static boolean isDockerImagePresent(String imageName) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("docker", "image", "inspect", imageName); + Process start = pb.start(); + return start.waitFor() == 0; - public static String whoIsMaster(){ - try { - InputStream pb = new ProcessBuilder("bash", "-c", "cat /etc/slurm/slurm.conf | grep ControlMachine").start().getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); - return reader.readLine(); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public static String whoAmI(){ - try { - InputStream pb = new ProcessBuilder("hostname").start().getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); - return reader.readLine(); - - } catch (IOException e) { - throw new RuntimeException(e); - } } - public static boolean pullSifImagefromRemoteDockerRepo(String imagePath, String imageName) { - if (imagePath == null || imagePath.isEmpty()) { + public static boolean pullSifImagefromRemoteDockerRepo(String dockerImageName, String sifImagePath) { + if (sifImagePath == null || sifImagePath.isEmpty()) { throw new IllegalArgumentException("imagePath is null or empty"); } @@ -52,8 +35,8 @@ public static boolean pullSifImagefromRemoteDockerRepo(String imagePath, String List command = List.of( "apptainer", "build", - imageName + ".sif", - "docker://" + imagePath + sifImagePath, + "docker://" + dockerImageName ); ProcessBuilder pb = new ProcessBuilder(command); pb.redirectErrorStream(true); @@ -68,17 +51,17 @@ public static boolean pullSifImagefromRemoteDockerRepo(String imagePath, String } - public static boolean pullSifImagefromLocalDockerRepo(String imagePath, String imageName) { - if (imagePath == null || imagePath.isEmpty()) { - throw new IllegalArgumentException("imagePath is null or empty"); + public static boolean pullSifImagefromLocalDockerRepo(String dockerImageName, String sifImagePath) { + if (dockerImageName == null || dockerImageName.isEmpty()) { + throw new IllegalArgumentException("dockerImageName is null or empty"); } System.out.println("Correct format: apptainer build {Sif_image_name}.sif docker-daemon://{image:tag}"); List command = List.of( "apptainer", "build", - imageName + ".sif", - "docker-daemon://" + imagePath + sifImagePath, + "docker-daemon://" + dockerImageName ); ProcessBuilder pb = new ProcessBuilder(command); @@ -94,179 +77,127 @@ public static boolean pullSifImagefromLocalDockerRepo(String imagePath, String i } - public static boolean submitHeldJob(String scriptPath) throws IOException, InterruptedException { - ProcessBuilder pb = new ProcessBuilder("sbatch", "--hold", scriptPath); - pb.redirectErrorStream(true); - Process process = pb.start(); - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line; - while ((line = reader.readLine()) != null) { - if (line.startsWith("Submitted batch job")) { - return true; - } - } - } - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("sbatch submission failed"); - } - return false; + public static JSONObject generateJobScript_GPU( + String jobname, + String runtime, + String workdir, + String hostPort, + String sif_location, + String uvicornCmd, + String memory, + String gpu, + String sifimagename, + String partitionname, + String whichnode, + String cpus + ) { + int minutes = Integer.parseInt(runtime); + int mems = Integer.parseInt(memory); + + int outer = Integer.parseInt(hostPort); + JSONObject job = new JSONObject() + .put("name", jobname) + .put("partition", partitionname) + .put("time_limit", minutes) + .put("current_working_directory", workdir) + .put("cpus_per_task", Integer.parseInt(cpus)) + .put("nodes", "1") + .put("required_nodes", List.of(whichnode)) + .put("memory_per_node", new JSONObject() + .put("set", true) + .put("number", mems)) + .put( "tres_per_node", "gres/gpu="+gpu) + .put("standard_output", workdir+"/"+sifimagename+"/"+sifimagename+".out") + .put("standard_error", workdir+"/"+sifimagename+"/"+sifimagename+".err") + .put("environment", + new JSONArray().put("PATH=/bin:/usr/bin:/sbin:$PATH")); + + String script = """ + #!/bin/bash -e + find_free() { for p in $(seq 30000 31000); do ss -ltn "( sport = :$p )" | grep -q LISTEN || { echo $p; return; }; done; } + INNER=$(find_free) + PORT=%d + IMG="%s" + INTOIMAGE="cd /usr/src/app" + UVI="%s" + + apptainer exec "$IMG" \\ + sh -c "$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER" & + PID=$! + + socat TCP-LISTEN:$PORT,bind=0.0.0.0,reuseaddr,fork TCP:127.0.0.1:$INNER & + PID_SOCAT=$! + + trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT + wait $PID + """.formatted(outer, sif_location, uvicornCmd); + + return new JSONObject() + .put("script", script) + .put("job", job); } -// public static void releaseJob(String jobId) throws IOException, InterruptedException { -// ProcessBuilder pb = new ProcessBuilder("scontrol", "release", jobId); -// pb.environment().put("SLURM_JOB_ID", jobId); -// pb.inheritIO(); -// Process process = pb.start(); -// int exitCode = process.waitFor(); -// if (exitCode != 0) { -// throw new RuntimeException("Failed to release job " + jobId); -// } -// } - - - public static List getHeldJobs() throws IOException, InterruptedException { - List jobIds = new ArrayList<>(); - ProcessBuilder pb = new ProcessBuilder("squeue", "--user=" + System.getProperty("user.name"), "--state=PD", "-h", "-o", "%i"); - Process process = pb.start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line; - while ((line = reader.readLine()) != null) { - jobIds.add(Integer.parseInt(line.trim())); - } - } - - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("Failed to fetch held jobs"); - } - - return jobIds; - } -// -// public static void releaseAllHeldJobs() throws IOException, InterruptedException { -// List heldJobs = getHeldJobs(); -// for (int jobId : heldJobs) { -// releaseJob(jobId); -// } -// } -// - - - - public static void squeue() throws IOException { - ProcessBuilder pb = new ProcessBuilder("squeue"); - pb.inheritIO(); - pb.start(); - } - - - public static void cancelJob(int jobId) throws IOException, InterruptedException { - ProcessBuilder pb = new ProcessBuilder("scancel", String.valueOf(jobId)); - pb.inheritIO(); - Process process = pb.start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new RuntimeException("Failed to cancel job " + jobId); - } + public static JSONObject generateJobScript_non_GPU( + String jobname, + String runtime, + String workdir, + String hostPort, + String sif_location, + String uvicornCmd, + String memory, + // String gpu, + String sifimagename, + String partitionname, + String whichnode, + String cpus + ) { + int minutes = Integer.parseInt(runtime); + int mems = Integer.parseInt(memory); + int outer = Integer.parseInt(hostPort); + JSONObject job = new JSONObject() + .put("name", jobname) + .put("partition", partitionname) + .put("time_limit", minutes) + .put("cpus_per_task", Integer.parseInt(cpus)) + .put("current_working_directory", workdir) + .put("nodes", "1") + .put("required_nodes", List.of(whichnode)) + .put("memory_per_node", new JSONObject() + .put("set", true) + .put("number", mems)) + .put("standard_output", workdir+"/"+sifimagename+"/"+sifimagename+".out") + .put("standard_error", workdir+"/"+sifimagename+"/"+sifimagename+".err") + .put("environment", + new JSONArray().put("PATH=/bin:/usr/bin:/sbin:$PATH")); + + String script = """ + #!/bin/bash -e + find_free() { for p in $(seq 30000 31000); do ss -ltn "( sport = :$p )" | grep -q LISTEN || { echo $p; return; }; done; } + INNER=$(find_free) + PORT=%d + IMG="%s" + INTOIMAGE="cd /usr/src/app" + UVI="%s" + + apptainer exec "$IMG" \\ + sh -c "$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER" & + PID=$! + + socat TCP-LISTEN:$PORT,bind=0.0.0.0,reuseaddr,fork TCP:127.0.0.1:$INNER & + PID_SOCAT=$! + + trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT + wait $PID + """.formatted(outer, sif_location, uvicornCmd); + + // ---------- 3. payload ---------- + return new JSONObject() + .put("script", script) + .put("job", job); } - - - // Ich finde das ziemlich zeitaufwendig, deshalb möchte ich später eine ähnliche Webseite für duui entwickeln - //Zum Beispiel könnte man den Namen einer DUUI-Komponente eingeben, und das System würde automatisch eine passende - // Konfiguration mit Entrypoint vorschlagen oder vervollständigen. - public static String submitJob(String jobName,String hostport, - String cpus,String gpu,String entry, String error, String imageport,String mem, - String time, - String output,String sifname, String saveIn, String uvicorn - ) throws IOException, InterruptedException { - - Path script = Paths.get("/tmp", jobName + ".sh"); - - - List lines = List.of( - "#!/bin/bash", - "#SBATCH --job-name=" + jobName, - "#SBATCH --cpus-per-task=" + cpus, - "#SBATCH --time=" + time, - "", - "PORT=" + hostport, - "UVI=\"" + uvicorn + "\"", // - "INNER=" + imageport, - "IMG=\"" + saveIn + "\"", - "INTOIMAGE=\"" + entry + "\"", // looks like "cd /usr/src/app" - "", - "apptainer exec \"$IMG\" \\", - " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &", - "PID=$!", - "", - "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &", - "PID_SOCAT=$!", - "", - "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT", - "", - "wait $PID" - ); - - - Files.write(script, lines, StandardCharsets.UTF_8, - StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - script.toFile().setExecutable(true); - System.out.println("Slurm batch script written to: " + script); - - - ProcessBuilder pb = new ProcessBuilder("sbatch", "--parsable",script.toString()); - pb.redirectErrorStream(true); - Process proc = pb.start(); - - String jobId; - try (BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()))) { - jobId = br.readLine(); - } - int exit = proc.waitFor(); - if (exit != 0 || jobId == null || jobId.isBlank()) { - throw new IllegalStateException("sbatch failed，exit=" + exit); - } - - System.out.println("Job submitted. ID = " + jobId); - return jobId.trim(); - } - - public static boolean checkSocatInstalled(){ - try { - InputStream pb = new ProcessBuilder("which", "socat").start().getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); - String s = reader.readLine(); - System.out.println(s); - return !s.isEmpty(); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - public static boolean checkSlurmInstalled(){ - try { - InputStream pb = new ProcessBuilder("slurmd", " ", "-C").start().getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(pb)); - String s = reader.readLine(); - System.out.println(s); - return !s.isEmpty(); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - - - - - - } diff --git a/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java new file mode 100755 index 00000000..fcb0cc95 --- /dev/null +++ b/src/main/java/org/texttechnologylab/DockerUnifiedUIMAInterface/driver/slurm/slurmInDocker/SlurmRest.java @@ -0,0 +1,237 @@ +package org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.async.ResultCallback; +import com.github.dockerjava.api.command.ExecCreateCmdResponse; +import com.github.dockerjava.api.model.Container; +import com.github.dockerjava.api.model.Frame; +import com.github.dockerjava.core.DockerClientBuilder; +import okhttp3.*; +import okhttp3.logging.HttpLoggingInterceptor; +import org.javatuples.Tuple; +import org.json.JSONObject; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class SlurmRest { + private DockerClient dockerClient = DockerClientBuilder.getInstance().build(); + private final String RESTVERSION = "v0.0.42"; + private final String PRE_URL; + private final String URL; + OkHttpClient httpClient = new OkHttpClient(); + + /** + * + * @param preUrl if slurmrestd on host preurl is http:localhost + */ + public SlurmRest(String preUrl) { + PRE_URL = preUrl; + URL = PRE_URL + ":6820/slurm/" + RESTVERSION + "/"; + } + + /** + * + * @return container name -- docker container id + */ + public Map containerNameID() { + Map nameID = new HashMap<>(); + List exec = dockerClient.listContainersCmd().exec(); + exec.stream().forEach((c) -> { + String[] names = c.getNames();// [/xxx] + //System.out.println(names[0]); + Pattern p = Pattern.compile("^/([a-zA-Z0-9]*)$"); + Matcher m = p.matcher(names[0]); + String result = m.find() ? m.group(1) : ""; + String id = c.getId(); + nameID.put(result, id); + }); + return nameID; + } + + /** + * + * @return all containers names + */ + public List listContainerNames() { + List exec = dockerClient.listContainersCmd().exec(); + List containers = new ArrayList<>(); + exec.stream().forEach(container -> { + containers.add(Arrays.toString(container.getNames())); + }); + return containers; + } + + /** + * + * @return check restd as container running in cluster + */ + public boolean checkRESTD() { + List containers = listContainerNames(); + return containers.stream().anyMatch(containerName -> containerName.contains("rest")); + } + + /** + * if the administrator has registered your user with the compute node, you can use your local name, + * * otherwise you can only use it as root + * @return + * @throws IOException + */ + public String showHostName() throws IOException { + ProcessBuilder pb = new ProcessBuilder(); + pb.command("whoami"); + Process start = pb.redirectErrorStream(true).start(); + BufferedReader br = new BufferedReader(new InputStreamReader(start.getInputStream())); + return br.readLine(); + } + + /** + * if the administrator has registered your user with the compute node, you can use your local name, otherwise you can only use it as root + * @param containerName Nodes that can generate jwt tokens for users include compute and slurmctld + * @return + * @throws IOException + * @throws InterruptedException + */ + public String generateTokenByHost(String containerName) throws IOException, InterruptedException { + String hostName = showHostName(); + String arg = "username=".concat(hostName); + String[] comms = new String[]{"scontrol", "token", arg}; + // default life-time is 5min + String token = executeInContainer(containerName, comms); + String[] split = token.split("=", 2); + return split[1].trim(); + } + + /** + * if the administrator has registered your user with the compute node, you can use your local name, otherwise you can only use it as root + * @param containerName Nodes that can generate jwt tokens for users include compute and slurmctld + * @return + * @throws IOException + * @throws InterruptedException + */ + public String generateRootToken(String containerName) throws IOException, InterruptedException { + String arg = "username=".concat("root"); + String[] comms = new String[]{"scontrol", "token", arg}; + String token = executeInContainer(containerName, comms); + String[] split = token.split("=", 2); + return split[1].trim(); + } + + public String executeInContainer(String containerName, String[] commands) throws IOException, InterruptedException { + Map nameIDMap = containerNameID(); + String containerId = nameIDMap.get(containerName); + ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(containerId) + .withAttachStdout(true) + .withAttachStderr(true) + .withCmd(commands) + .exec(); + + StringBuilder result = new StringBuilder(); + dockerClient.execStartCmd(execCreateCmdResponse.getId()) + .exec(new ResultCallback.Adapter() { + @Override + public void onNext(Frame frame) { + result.append(new String(frame.getPayload())); + } + }).awaitCompletion(); + return result.toString(); + } + + /** + * + * @param token jwt password + * @param where Specify permitted format queries here, https://slurm.schedmd.com/rest.html + * @return + * @throws IOException + * @throws InterruptedException + */ + public String query(String token, String where) throws IOException, InterruptedException { + Request req = new Request.Builder().url(URL.concat(where)).header("X-SLURM-USER-TOKEN", token).get().build(); + try (Response response = httpClient.newCall(req).execute()) { + return response.body().string(); + } + } + + /** + * Specify permitted format queries here, https://slurm.schedmd.com/rest.html + * @param token + * @param params + * @return [0]:full json response, [1]: job id + * @throws IOException + * @throws InterruptedException + */ + public String[] submit(String token, JSONObject params) throws IOException, InterruptedException { + MediaType type = MediaType.get("application/json; charset=utf-8"); + RequestBody requestBody = RequestBody.create(null, params.toString()); + + Request req = new Request.Builder().url(URL.concat("job/submit")). + addHeader("X-SLURM-USER-TOKEN", token). + addHeader("Content-Type", "application/json"). + post(requestBody). + build(); + try (Response response = httpClient.newCall(req).execute()) { + if (!response.isSuccessful()) { + throw new IOException("HTTP " + response.code() + " - " + response.message() + + "\n" + response.body().string()); + } + + String respStr = response.body().string(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(respStr); + String jobId = jsonNode.get("job_id").asText(); + return new String[]{respStr, jobId}; + + } + } + + /** + * cancel x-th job + * @param token + * @param jobID + * @return + * @throws IOException + */ + public boolean cancelJob(String token, String jobID) throws IOException { + String cancelUrl = "job/".concat(jobID); + Request req = new Request.Builder() + .url(URL.concat(cancelUrl)) + .addHeader("X-SLURM-USER-TOKEN", token) + .delete() + .build(); + try (Response response = httpClient.newCall(req).execute()) { + if (!response.isSuccessful()) { + throw new IOException("HTTP " + response.code() + " - " + response.message() + + "\n" + response.body().string()); + } + String respStr = response.body().string(); + //System.out.println(respStr); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(respStr); + if (!jsonNode.hasNonNull("error")){ + System.out.println("[SlurmDriver] Cancel "+jobID+" Successfully"); + return true; + } + else if(jsonNode.get("status").get("error").get("code").asInt()==2021){ + System.out.println("[SlurmDriver] Cancel "+jobID+" Successfully"); + return true; + } + else { + System.out.println("[SlurmDriver] Failed to Cancel "+jobID+" , see logs"); + return false; + } + } + } +} + + diff --git a/src/test/java/SlurmTests.java b/src/test/java/SlurmTests.java deleted file mode 100644 index 694a5604..00000000 --- a/src/test/java/SlurmTests.java +++ /dev/null @@ -1,68 +0,0 @@ -import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.io.xmi.XmiWriter; -import org.junit.jupiter.api.Test; -import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver; -import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; -import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaSandbox; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; - -public class SlurmTests { - @Test - public void basicTest() throws Exception { - int iWorkers = 1; // - JCas jc = JCasFactory.createText("hello world"); - DocumentMetaData dmd = new DocumentMetaData(jc); - dmd.setDocumentId("test05"); - dmd.setDocumentTitle("univers test05"); - dmd.addToIndexes(); - DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); - DUUILuaSandbox sandbox = new DUUILuaSandbox(); - sandbox._allowAllJavaClasses = true; - ctx.withSandbox(sandbox); - DUUIComposer composer = new DUUIComposer() - .withSkipVerification(true) - .withLuaContext(ctx) - .withWorkers(iWorkers); - DUUIUIMADriver uimaDriver = new DUUIUIMADriver(); - // driver hinzufügen - DUUISlurmDriver slurmDriver = new DUUISlurmDriver(); - - composer.addDriver(uimaDriver, slurmDriver); - - composer.resetPipeline(); - DUUIPipelineComponent com1 = new DUUISlurmDriver.Component(new DUUIPipelineComponent() - .withSlurmCpus("2") - .withSlurmGPU("0") - .withSlurmMemory("1G") - .withSlurmImagePort("9714") - .withSlurmJobName("test1") - .withSlurmEntryLocation("cd /usr/src/app") - .withSlurmErrorLocation("/tmp") - .withSlurmSaveIn("/home/jd/again.sif") - .withSlurmRuntime("1:00:00") - .withScale(1) - .withSlurmUvicorn("uvicorn textimager_duui_spacy:app") - ).build(); - - - composer.add(com1); - - composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, - XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/", - XmiWriter.PARAM_PRETTY_PRINT, true, - XmiWriter.PARAM_OVERWRITE, true, - XmiWriter.PARAM_VERSION, "1.1", - XmiWriter.PARAM_COMPRESSION, "GZIP")) - .build()); - - composer.run(jc); - - - } -} diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java index cc7bb423..cf721df4 100644 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java +++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/io/abbyy/TestAbbyyDocumentReader.java @@ -7,7 +7,7 @@ import org.apache.uima.fit.factory.CollectionReaderFactory; import org.apache.uima.fit.pipeline.SimplePipeline; import org.dkpro.core.io.xmi.XmiWriter; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.texttechnologylab.DockerUnifiedUIMAInterface.io.reader.abbyy.AbbyyDocumentReader; import java.io.IOException; diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java index 6f12bab0..ed1f5fe6 100644 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java +++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestGNFinderLite.java @@ -1,130 +1,130 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; - -import org.apache.uima.cas.CASException; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; -import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; -import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; -import org.texttechnologylab.annotation.biofid.gnfinder.MetaData; -import org.texttechnologylab.annotation.biofid.gnfinder.Taxon; - -import java.io.IOException; -import java.net.URISyntaxException; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class TestGNFinderLite { - final DockerTestContainerManager container = new DockerTestContainerManager( - "docker.texttechnologylab.org/duui-lite-gnfinder:0.1.1" - ); - - DUUIComposer composer; - - @BeforeAll - public void setUp() throws Exception { - composer = new DUUIComposer() - .withLuaContext( - new DUUILuaContext() - .withJsonLibrary()) - .withSkipVerification(true); - composer.addDriver(new DUUIRemoteDriver(10000)); - } - - @AfterAll - public void shutdown() throws Exception { - if (composer != null) - composer.shutdown(); - - container.close(); - } - - private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException { - return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())); - } - - private static JCas getJCas() throws ResourceInitializationException, CASException { - JCas jCas = JCasFactory.createJCas(); - jCas.setDocumentText( - String.join("\n", "Nach Schluß des Congresses ist eine längere Excursion vorgesehen, auf welcher die Inseln an der Küste von Pembrokshire besucht werden.", "Dieser Ausflug dürfte besonders interessant werden, weil sich hier große Brutkolonien von Puffinus puffinus und verschiedener Alcidae befinden.", "Auch Thalassidroma pelagica dürfte hier angetroffen werden.", "Bei günstigem Wetter ist ferner der Besuch einer Brutkolonie von Sula bassana vorgesehen.", "Homo sapiens sapiens.")); - jCas.setDocumentLanguage("de"); - return jCas; - } - - @Test - public void test_default() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - .build() - ); - - JCas jCas = getJCas(); - composer.run(jCas); - - printResults(jCas); - } - - @Test - public void test_with_noBayes() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - .withParameter("noBayes", "true") - .build() - ); - - JCas jCas = getJCas(); - composer.run(jCas); - - printResults(jCas); - } - - @Test - public void test_with_allMatches() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - // Catalogue of Life and GBIF - .withParameter("sources", "[1, 11]") - .withParameter("allMatches", "true") - .build() - ); - - JCas jCas = getJCas(); - composer.run(jCas); - - printResults(jCas); - } - - @Test - public void test_with_oddsDetails() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - .withParameter("oddsDetails", "true") - .build() - ); - - JCas jCas = getJCas(); - composer.run(jCas); - - printResults(jCas); - } - - private static void printResults(JCas jCas) { - System.out.println(jCas.select(MetaData.class).findFirst().get().toString(2)); - System.out.println(); - - for (Taxon tx : jCas.select(Taxon.class)) { - System.out.print(tx.toString(2)); - System.out.println("\n > coveredText: \"" + tx.getCoveredText() + "\"\n"); - System.out.println(); - } - } -} +//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; +// +//import org.apache.uima.cas.CASException; +//import org.apache.uima.fit.factory.JCasFactory; +//import org.apache.uima.jcas.JCas; +//import org.apache.uima.resource.ResourceInitializationException; +//import org.junit.jupiter.api.AfterAll; +//import org.junit.jupiter.api.BeforeAll; +//import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.TestInstance; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +//import org.texttechnologylab.annotation.biofid.gnfinder.MetaData; +//import org.texttechnologylab.annotation.biofid.gnfinder.Taxon; +// +//import java.io.IOException; +//import java.net.URISyntaxException; +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//public class TestGNFinderLite { +// final DockerTestContainerManager container = new DockerTestContainerManager( +// "docker.texttechnologylab.org/duui-lite-gnfinder:0.1.1" +// ); +// +// DUUIComposer composer; +// +// @BeforeAll +// public void setUp() throws Exception { +// composer = new DUUIComposer() +// .withLuaContext( +// new DUUILuaContext() +// .withJsonLibrary()) +// .withSkipVerification(true); +// composer.addDriver(new DUUIRemoteDriver(10000)); +// } +// +// @AfterAll +// public void shutdown() throws Exception { +// if (composer != null) +// composer.shutdown(); +// +// container.close(); +// } +// +// private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException { +// return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())); +// } +// +// private static JCas getJCas() throws ResourceInitializationException, CASException { +// JCas jCas = JCasFactory.createJCas(); +// jCas.setDocumentText( +// String.join("\n", "Nach Schluß des Congresses ist eine längere Excursion vorgesehen, auf welcher die Inseln an der Küste von Pembrokshire besucht werden.", "Dieser Ausflug dürfte besonders interessant werden, weil sich hier große Brutkolonien von Puffinus puffinus und verschiedener Alcidae befinden.", "Auch Thalassidroma pelagica dürfte hier angetroffen werden.", "Bei günstigem Wetter ist ferner der Besuch einer Brutkolonie von Sula bassana vorgesehen.", "Homo sapiens sapiens.")); +// jCas.setDocumentLanguage("de"); +// return jCas; +// } +// +// @Test +// public void test_default() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// .build() +// ); +// +// JCas jCas = getJCas(); +// composer.run(jCas); +// +// printResults(jCas); +// } +// +// @Test +// public void test_with_noBayes() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// .withParameter("noBayes", "true") +// .build() +// ); +// +// JCas jCas = getJCas(); +// composer.run(jCas); +// +// printResults(jCas); +// } +// +// @Test +// public void test_with_allMatches() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// // Catalogue of Life and GBIF +// .withParameter("sources", "[1, 11]") +// .withParameter("allMatches", "true") +// .build() +// ); +// +// JCas jCas = getJCas(); +// composer.run(jCas); +// +// printResults(jCas); +// } +// +// @Test +// public void test_with_oddsDetails() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// .withParameter("oddsDetails", "true") +// .build() +// ); +// +// JCas jCas = getJCas(); +// composer.run(jCas); +// +// printResults(jCas); +// } +// +// private static void printResults(JCas jCas) { +// System.out.println(jCas.select(MetaData.class).findFirst().get().toString(2)); +// System.out.println(); +// +// for (Taxon tx : jCas.select(Taxon.class)) { +// System.out.print(tx.toString(2)); +// System.out.println("\n > coveredText: \"" + tx.getCoveredText() + "\"\n"); +// System.out.println(); +// } +// } +//} diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java index 12096893..1e7df5fd 100644 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java +++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyBatched.java @@ -1,146 +1,146 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; - -import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.MorphologicalFeatures; -import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; -import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import org.apache.uima.cas.CASException; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.fit.util.JCasUtil; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; -import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; -import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; -import org.texttechnologylab.annotation.SpacyAnnotatorMetaData; - -import java.io.IOException; -import java.net.URISyntaxException; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class TestSpacyBatched { - final DockerTestContainerManager container = new DockerTestContainerManager( - "docker.texttechnologylab.org/duui-spacy-lua-process:0.1.0", - 6000 - ); - - DUUIComposer composer; - - @BeforeAll - public void setUp() throws Exception { - composer = new DUUIComposer() - .withLuaContext( - new DUUILuaContext() - .withJsonLibrary()) - .withSkipVerification(true); - composer.addDriver(new DUUIRemoteDriver(10000)); - } - - @AfterAll - public void shutdown() throws Exception { - if (composer != null) - composer.shutdown(); - - container.close(); - } - - private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException { - return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())); - } - - private static JCas getJCas() throws ResourceInitializationException, CASException { - JCas jCas = JCasFactory.createJCas(); - jCas.setDocumentText( - "Die Goethe Universität ist auf vier große Universitätsgelände über das Frankfurter Stadtgebiet verteilt.\n " - + "Barack Obama war der 44. Präsident der Vereinigten Staaten von Amerika." - ); - jCas.setDocumentLanguage("de"); - return jCas; - } - - @Test - public void test_with_sentences() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - .withParameter("spacy_model_size", "lg") - .build() - ); - - JCas jCas = getJCas(); - new Sentence(jCas, 0, 104).addToIndexes(); - new Sentence(jCas, 106, 177).addToIndexes(); - - composer.run(jCas, "lua-process-test/w-sentences"); - - printResult(jCas); - } - - @Test - public void test_wo_sentences() throws Exception { - composer.resetPipeline(); - composer.add( - getComponent() - .withParameter("spacy_model_size", "sm") - .build() - ); - - JCas jCas = getJCas(); - - composer.run(jCas, "lua-process-test/wo-sentences"); - - printResult(jCas); - } - - private static void printResult(JCas jCas) { - System.out.println("### SpacyAnnotatorMetaData ###"); - for (SpacyAnnotatorMetaData annotation : JCasUtil.select(jCas, SpacyAnnotatorMetaData.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### Sentence ###"); - for (Sentence annotation : JCasUtil.select(jCas, Sentence.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### Token ###"); - for (Token annotation : JCasUtil.select(jCas, Token.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### Lemma ###"); - for (Lemma annotation : JCasUtil.select(jCas, Lemma.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### POS ###"); - for (POS annotation : JCasUtil.select(jCas, POS.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### MorphologicalFeatures ###"); - for (MorphologicalFeatures annotation : JCasUtil.select(jCas, MorphologicalFeatures.class)) { - System.out.println(annotation.toString(2)); - System.out.println(); - } - - System.out.println("### NamedEntity ###"); - for (NamedEntity annotation : JCasUtil.select(jCas, NamedEntity.class)) { - System.out.print(annotation.toString(2)); - System.out.printf("%n text: '%s'%n", annotation.getCoveredText()); - System.out.println(); - } - } -} +//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; +// +//import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.MorphologicalFeatures; +//import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; +//import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; +//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma; +//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; +//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; +//import org.apache.uima.cas.CASException; +//import org.apache.uima.fit.factory.JCasFactory; +//import org.apache.uima.fit.util.JCasUtil; +//import org.apache.uima.jcas.JCas; +//import org.apache.uima.resource.ResourceInitializationException; +//import org.junit.jupiter.api.AfterAll; +//import org.junit.jupiter.api.BeforeAll; +//import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.TestInstance; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +//import org.texttechnologylab.annotation.SpacyAnnotatorMetaData; +// +//import java.io.IOException; +//import java.net.URISyntaxException; +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//public class TestSpacyBatched { +// final DockerTestContainerManager container = new DockerTestContainerManager( +// "docker.texttechnologylab.org/duui-spacy-lua-process:0.1.0", +// 6000 +// ); +// +// DUUIComposer composer; +// +// @BeforeAll +// public void setUp() throws Exception { +// composer = new DUUIComposer() +// .withLuaContext( +// new DUUILuaContext() +// .withJsonLibrary()) +// .withSkipVerification(true); +// composer.addDriver(new DUUIRemoteDriver(10000)); +// } +// +// @AfterAll +// public void shutdown() throws Exception { +// if (composer != null) +// composer.shutdown(); +// +// container.close(); +// } +// +// private DUUIRemoteDriver.Component getComponent() throws URISyntaxException, IOException { +// return new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort())); +// } +// +// private static JCas getJCas() throws ResourceInitializationException, CASException { +// JCas jCas = JCasFactory.createJCas(); +// jCas.setDocumentText( +// "Die Goethe Universität ist auf vier große Universitätsgelände über das Frankfurter Stadtgebiet verteilt.\n " +// + "Barack Obama war der 44. Präsident der Vereinigten Staaten von Amerika." +// ); +// jCas.setDocumentLanguage("de"); +// return jCas; +// } +// +// @Test +// public void test_with_sentences() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// .withParameter("spacy_model_size", "lg") +// .build() +// ); +// +// JCas jCas = getJCas(); +// new Sentence(jCas, 0, 104).addToIndexes(); +// new Sentence(jCas, 106, 177).addToIndexes(); +// +// composer.run(jCas, "lua-process-test/w-sentences"); +// +// printResult(jCas); +// } +// +// @Test +// public void test_wo_sentences() throws Exception { +// composer.resetPipeline(); +// composer.add( +// getComponent() +// .withParameter("spacy_model_size", "sm") +// .build() +// ); +// +// JCas jCas = getJCas(); +// +// composer.run(jCas, "lua-process-test/wo-sentences"); +// +// printResult(jCas); +// } +// +// private static void printResult(JCas jCas) { +// System.out.println("### SpacyAnnotatorMetaData ###"); +// for (SpacyAnnotatorMetaData annotation : JCasUtil.select(jCas, SpacyAnnotatorMetaData.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### Sentence ###"); +// for (Sentence annotation : JCasUtil.select(jCas, Sentence.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### Token ###"); +// for (Token annotation : JCasUtil.select(jCas, Token.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### Lemma ###"); +// for (Lemma annotation : JCasUtil.select(jCas, Lemma.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### POS ###"); +// for (POS annotation : JCasUtil.select(jCas, POS.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### MorphologicalFeatures ###"); +// for (MorphologicalFeatures annotation : JCasUtil.select(jCas, MorphologicalFeatures.class)) { +// System.out.println(annotation.toString(2)); +// System.out.println(); +// } +// +// System.out.println("### NamedEntity ###"); +// for (NamedEntity annotation : JCasUtil.select(jCas, NamedEntity.class)) { +// System.out.print(annotation.toString(2)); +// System.out.printf("%n text: '%s'%n", annotation.getCoveredText()); +// System.out.println(); +// } +// } +//} diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java index f3104757..a55a2bcc 100644 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java +++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/lua_process/TestSpacyEos.java @@ -1,88 +1,88 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import org.apache.uima.cas.CASException; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.fit.util.JCasUtil; -import org.apache.uima.jcas.JCas; -import org.apache.uima.jcas.cas.TOP; -import org.apache.uima.resource.ResourceInitializationException; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; -import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; -import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; -import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class TestSpacyEos { - final DockerTestContainerManager container = new DockerTestContainerManager( - "docker.texttechnologylab.org/duui-spacy-eos:0.1.0", - 6000 - ); - - DUUIComposer composer; - - @BeforeAll - public void setUp() throws Exception { - composer = new DUUIComposer() - .withLuaContext( - new DUUILuaContext() - .withJsonLibrary()) - .withSkipVerification(true); - composer.addDriver(new DUUIRemoteDriver(10000)); - composer.add(new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()))); - } - - @AfterAll - public void shutdown() throws Exception { - if (composer != null) - composer.shutdown(); - - container.close(); - } - - @Test - public void testEnglish() throws Exception { - JCas jCas = getJCas(); - jCas.setDocumentLanguage("en"); - - composer.run(jCas, "duui-spacy-eos:0.1.0/en"); - - printResults(jCas); - } - - @NotNull - private static JCas getJCas() throws ResourceInitializationException, CASException { - JCas jCas = JCasFactory.createJCas(); - jCas.setDocumentText( - "Barack Hussein Obama II (born August 4, 1961) is an American politician who was the 44th president of the United States from 2009 to 2017. " - + "A member of the Democratic Party, he was the first African-American president in American history. " - + "Obama previously served as a U.S. senator representing Illinois from 2005 to 2008 and as an Illinois state senator from 1997 to 2004." - ); - return jCas; - } - - @Test - public void testNoLang() throws Exception { - JCas jCas = getJCas(); - - composer.run(jCas, "duui-spacy-eos:0.1.0/no-lang"); - - printResults(jCas); - } - - private static void printResults(JCas jCas) { - for (TOP annotation : JCasUtil.select(jCas, TOP.class)) { - System.out.print(annotation.toString(2)); - if (annotation instanceof Sentence) { - System.out.printf("%n text: '%s'%n%n", ((Sentence) annotation).getCoveredText()); - } else { - System.out.printf("%n%n"); - } - } - } -} \ No newline at end of file +//package org.texttechnologylab.DockerUnifiedUIMAInterface.lua_process; +// +//import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; +//import org.apache.uima.cas.CASException; +//import org.apache.uima.fit.factory.JCasFactory; +//import org.apache.uima.fit.util.JCasUtil; +//import org.apache.uima.jcas.JCas; +//import org.apache.uima.jcas.cas.TOP; +//import org.apache.uima.resource.ResourceInitializationException; +//import org.jetbrains.annotations.NotNull; +//import org.junit.jupiter.api.AfterAll; +//import org.junit.jupiter.api.BeforeAll; +//import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.TestInstance; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.test.DockerTestContainerManager; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIRemoteDriver; +//import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +// +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//public class TestSpacyEos { +// final DockerTestContainerManager container = new DockerTestContainerManager( +// "docker.texttechnologylab.org/duui-spacy-eos:0.1.0", +// 6000 +// ); +// +// DUUIComposer composer; +// +// @BeforeAll +// public void setUp() throws Exception { +// composer = new DUUIComposer() +// .withLuaContext( +// new DUUILuaContext() +// .withJsonLibrary()) +// .withSkipVerification(true); +// composer.addDriver(new DUUIRemoteDriver(10000)); +// composer.add(new DUUIRemoteDriver.Component("http://localhost:%d".formatted(container.getPort()))); +// } +// +// @AfterAll +// public void shutdown() throws Exception { +// if (composer != null) +// composer.shutdown(); +// +// container.close(); +// } +// +// @Test +// public void testEnglish() throws Exception { +// JCas jCas = getJCas(); +// jCas.setDocumentLanguage("en"); +// +// composer.run(jCas, "duui-spacy-eos:0.1.0/en"); +// +// printResults(jCas); +// } +// +// @NotNull +// private static JCas getJCas() throws ResourceInitializationException, CASException { +// JCas jCas = JCasFactory.createJCas(); +// jCas.setDocumentText( +// "Barack Hussein Obama II (born August 4, 1961) is an American politician who was the 44th president of the United States from 2009 to 2017. " +// + "A member of the Democratic Party, he was the first African-American president in American history. " +// + "Obama previously served as a U.S. senator representing Illinois from 2005 to 2008 and as an Illinois state senator from 1997 to 2004." +// ); +// return jCas; +// } +// +// @Test +// public void testNoLang() throws Exception { +// JCas jCas = getJCas(); +// +// composer.run(jCas, "duui-spacy-eos:0.1.0/no-lang"); +// +// printResults(jCas); +// } +// +// private static void printResults(JCas jCas) { +// for (TOP annotation : JCasUtil.select(jCas, TOP.class)) { +// System.out.print(annotation.toString(2)); +// if (annotation instanceof Sentence) { +// System.out.printf("%n text: '%s'%n%n", ((Sentence) annotation).getCoveredText()); +// } else { +// System.out.printf("%n%n"); +// } +// } +// } +//} \ No newline at end of file diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java deleted file mode 100644 index ad03e7dc..00000000 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/test/DockerTestContainerManager.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.texttechnologylab.DockerUnifiedUIMAInterface.test; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.CreateContainerResponse; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.HostConfig; -import com.github.dockerjava.api.model.PortBinding; -import com.github.dockerjava.core.DockerClientBuilder; -import org.apache.commons.lang.math.IntRange; - -import java.io.IOException; -import java.net.ServerSocket; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -/** - * Test helper class to create and run a container image for testing purposes. - * Will search for a random free port - */ -public class DockerTestContainerManager implements AutoCloseable { - final DockerClient dockerClient = DockerClientBuilder.getInstance().build(); - final CreateContainerResponse container; - final int port = searchFreePort(); - - public DockerTestContainerManager(String imageName) { - this(imageName, "test-duui-" + UUID.randomUUID(), 5000); - } - - public DockerTestContainerManager(String imageName, String containerName) { - this(imageName, containerName, 5000); - } - - public DockerTestContainerManager(String imageName, long startupDelay) { - this(imageName, "test-duui-" + UUID.randomUUID(), startupDelay); - } - - public DockerTestContainerManager(String imageName, String containerName, long startupDelay) { - System.out.printf("[DockerTestContainerManager] Creating Container Image for %s%n", imageName); - container = dockerClient.createContainerCmd(imageName) - .withHostConfig( - HostConfig.newHostConfig() - .withAutoRemove(true) - .withPublishAllPorts(true) - .withPortBindings(PortBinding.parse("%d:9714".formatted(port))) - ) - .withExposedPorts(ExposedPort.tcp(9714)) - .withName(containerName) - .exec(); - System.out.printf("[DockerTestContainerManager] Container Image Created: %s%n", container.getId()); - System.out.printf("[DockerTestContainerManager] Starting Container as %s with port binding %d:9714%n", containerName, port); - dockerClient.startContainerCmd(container.getId()).exec(); - - try { - System.out.printf("[DockerTestContainerManager] Waiting %dms for Container to Come Online%n", startupDelay); - Thread.sleep(startupDelay); - } catch (InterruptedException e) { - } - - System.out.println("[DockerTestContainerManager] Container Started: " + container.getId()); - } - - private static int searchFreePort() { - List ports = Arrays.stream(new IntRange(30000, 40000).toArray()).boxed().collect(Collectors.toList()); - Collections.shuffle(ports); - for (int port : ports) { - try (ServerSocket socket = new ServerSocket(port)) { - if (socket.getLocalPort() == port) { - return port; - } - } catch (IOException e) { - if (!e.getMessage().contains("Address already in use")) { - throw new RuntimeException(e); - } - } - } - throw new RuntimeException("No free port found"); - } - - public int getPort() { - return port; - } - - @Override - public void close() throws Exception { - System.out.println("[DockerTestContainerManager] Stopping Container: " + container.getId()); - dockerClient.stopContainerCmd(container.getId()).withTimeout(10).exec(); - System.out.println("[DockerTestContainerManager] Container Stopped"); - System.out.println("[DockerTestContainerManager] Stopping Docker Client"); - dockerClient.close(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java index 7333847b..1dfdce5d 100644 --- a/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java +++ b/src/test/java/org/texttechnologylab/DockerUnifiedUIMAInterface/tools/TestAnnotationDropper.java @@ -86,7 +86,7 @@ static void resetCas() { } @AfterAll - static void afterAll() throws UnknownHostException { + static void afterAll() throws IOException, InterruptedException { composer.shutdown(); } diff --git a/src/test/java/slurmDriver/ParlbertTopicGermanTest.java b/src/test/java/slurmDriver/ParlbertTopicGermanTest.java new file mode 100755 index 00000000..19314c9d --- /dev/null +++ b/src/test/java/slurmDriver/ParlbertTopicGermanTest.java @@ -0,0 +1,147 @@ +package slurmDriver; + +import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; +import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; +import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; +import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; +import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; +import org.apache.uima.fit.factory.JCasFactory; +import org.apache.uima.fit.util.JCasUtil; +import org.apache.uima.jcas.JCas; +import org.apache.uima.util.XmlCasSerializer; +import org.dkpro.core.io.xmi.XmiWriter; +import org.junit.jupiter.api.Test; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.*; +import org.texttechnologylab.DockerUnifiedUIMAInterface.io.AsyncCollectionReader; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +import org.hucompute.textimager.uima.type.category.CategoryCoveredTagged; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +public class ParlbertTopicGermanTest { + + public static final String dijkstraExampleText = "Der Algorithmus von Dijkstra (nach seinem Erfinder Edsger W. Dijkstra) ist ein Algorithmus aus der Klasse der Greedy-Algorithmen[1] und löst das Problem der kürzesten Pfade für einen gegebenen Startknoten. " + + "Er berechnet somit einen kürzesten Pfad zwischen dem gegebenen Startknoten und einem der (oder allen) übrigen Knoten in einem kantengewichteten Graphen (sofern dieser keine Negativkanten enthält)." + + "Für unzusammenhängende ungerichtete Graphen ist der Abstand zu denjenigen Knoten unendlich, zu denen kein Pfad vom Startknoten aus existiert. Dasselbe gilt auch für gerichtete nicht stark zusammenhängende Graphen. Dabei wird der Abstand synonym auch als Entfernung, Kosten oder Gewicht bezeichnet."; + + @Test + public void test() throws Exception { + + int iWorkers = 1; + DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); + + // Instanziierung des Composers, mit einigen Parametern + DUUIComposer composer = new DUUIComposer() + .withSkipVerification(true) // wir überspringen die Verifikation aller Componenten =) + .withLuaContext(ctx) // wir setzen den definierten Kontext + .withWorkers(iWorkers); // wir geben dem Composer eine Anzahl an Threads mit. + + + DUUIDockerDriver docker_driver = new DUUIDockerDriver(); + DUUIRemoteDriver remoteDriver = new DUUIRemoteDriver(1000); + DUUIUIMADriver uima_driver = new DUUIUIMADriver() + .withDebug(false); + composer.addDriver(docker_driver, remoteDriver,uima_driver); + + boolean useDockerImage = false; + if (useDockerImage){ + composer.add(new DUUIDockerDriver.Component("docker.texttechnologylab.org/parlbert-topic-german:latest") + .withScale(iWorkers) + .build()); + }else{ + composer.add(new DUUIRemoteDriver.Component("http://localhost:20000") + .withScale(iWorkers) + .build()); + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/slurm", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1" + )).build()); + } + + // Create basic test jCas. + JCas jCas = JCasFactory.createText(dijkstraExampleText, "de"); + + new Sentence(jCas, 0, 206).addToIndexes(); + new Sentence(jCas, 206, 402).addToIndexes(); + new Sentence(jCas, 402, 544).addToIndexes(); + new Sentence(jCas, 544, 616).addToIndexes(); + new Sentence(jCas, 616, 699).addToIndexes(); + + composer.run(jCas, "test"); + + // Print Result + Collection categoryCoveredTaggeds = JCasUtil.select(jCas, CategoryCoveredTagged.class).stream().sorted((c1, c2) -> c1.getBegin()-c2.getBegin()).collect(Collectors.toList()); + for(CategoryCoveredTagged categoryCoveredTagged: categoryCoveredTaggeds){ + System.out.println(categoryCoveredTagged.getBegin() + " - " + categoryCoveredTagged.getEnd() + " " + categoryCoveredTagged.getValue() + ": " + categoryCoveredTagged.getScore()); + } + + + + } + + @Test + public void singleTokenizedTestDe() throws Exception { + DUUIComposer composer = new DUUIComposer() + .withLuaContext( + new DUUILuaContext() + .withJsonLibrary() + ) + .withSkipVerification(true); + + DUUIRemoteDriver remote_driver = new DUUIRemoteDriver(10000); + DUUIUIMADriver uima_driver = new DUUIUIMADriver(); + composer.addDriver(remote_driver,uima_driver); + + composer.add( + new DUUIRemoteDriver.Component("http://127.0.0.1:20000") + .withParameter("use_existing_tokens", String.valueOf(true)) + ); + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/slurm", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1" + )).build()); + + String language = "de"; + + JCas cas = JCasFactory.createJCas(); + cas.setDocumentText("Das ist ein IPhone von Apple. Und das ist ein iMac."); + cas.setDocumentLanguage(language); + + int[][] tokens1 = new int[][]{ + new int[]{0, 3}, //Das + new int[]{4, 7}, //ist + new int[]{8, 11}, //ein + new int[]{12, 18}, //IPhone + new int[]{19, 22}, //von + new int[]{23, 28}, //Apple + new int[]{28, 29}, //. + new int[]{30, 33}, //Und + new int[]{34, 37}, //das + new int[]{38, 41}, //ist + new int[]{42, 45}, //ein + new int[]{46, 50}, //iMac + new int[]{50, 51} //. + }; + + for (int[] tokenPos : tokens1) { + Token token = new Token(cas, tokenPos[0], tokenPos[1]); + token.addToIndexes(); + } + + composer.run(cas); + + } +} diff --git a/src/test/java/slurmDriver/RestApi.java b/src/test/java/slurmDriver/RestApi.java new file mode 100755 index 00000000..77a681be --- /dev/null +++ b/src/test/java/slurmDriver/RestApi.java @@ -0,0 +1,348 @@ +package slurmDriver; + +import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.uima.UIMAException; +import org.apache.uima.cas.CASException; +import org.apache.uima.fit.factory.JCasFactory; +import org.apache.uima.jcas.JCas; +import org.apache.uima.resource.ResourceInitializationException; +import org.dkpro.core.io.xmi.XmiWriter; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.SlurmUtils; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaContext; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.DUUILuaSandbox; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.List; + +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; + +public class RestApi { + SlurmRest slurmRest = new SlurmRest("http:localhost"); + + @Test + public void dockerCommandTest() throws IOException { + boolean b = slurmRest.checkRESTD(); + System.out.println(b); + System.out.println(slurmRest.showHostName()); + List strings = slurmRest.listContainerNames(); + System.out.println(strings); + + } + + @Test + public void seeInfos() throws IOException, InterruptedException { + // GET /slurm/v0.0.42/diag + // GET /slurm/v0.0.42/nodes + // GET /slurm/v0.0.42/jobs + + String s = slurmRest.generateRootToken("compute1"); + System.out.println("============================================================================="); + System.out.println(slurmRest.query(s, "diag")); + System.out.println("============================================================================="); + System.out.println(slurmRest.query(s, "jobs")); + System.out.println("============================================================================="); + + + } + + @Test + public void splitToken() { + String token = "SLURM_JWT=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NTE4ODY3MzIsImlhdCI6MTc1MTg4NDkzMiwic3VuIjoiamQifQ.uzplyXbsDPPix1hEU7JNcUhWUm8YPs6UHNTUpODLyE0"; + String[] split = token.split("=", 2); + split[0] = split[0].trim(); + split[1] = split[1].trim(); + System.out.println(split[0]); + System.out.println(split[1]); + } + + @Test + public void submitByRoot() throws IOException, InterruptedException { + JSONObject job = new JSONObject() + .put("name", "rest_smoke") + .put("partition", "normal") + .put("time_limit", 300) + .put("current_working_directory", "/data") + .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/")); + + JSONObject payload = new JSONObject() + .put("script", "#!/bin/bash\nsrun hostname") + .put("job", job); + + String token = slurmRest.generateRootToken("compute1"); + Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println); + } + + @Test + public void seeJobIDn() throws IOException, InterruptedException { + String s = slurmRest.generateRootToken("compute1"); + System.out.println("============================================================================="); + System.out.println(slurmRest.query(s, "job/13")); + } + + @Test + public void submitSpacy() throws IOException, InterruptedException { + + JSONObject job = new JSONObject() + .put("name", "spacy_test") + .put("partition", "normal") + .put("cpus_per_task", 2) + .put("required_nodes", List.of("compute1")) + .put( "tres_per_node", "gres/gpu=1") + .put("memory_per_node", new JSONObject() + .put("set", true) + .put("number", 2048)) + .put("time_limit", 600)//minute + .put("current_working_directory", "/data") + .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/")); + + JSONObject payload = new JSONObject() + .put("script", + "#!/bin/bash\n" + + "PORT=20000 \n" + + "INNER=9714 \n" + + "IMG=\"/sif_pool/spacy.sif\" " + + "INTOIMAGE=\"cd /usr/src/app\"\n" + + "UVI=\"uvicorn textimager_duui_spacy:app\" \n" + + "\n" + + "apptainer exec \"$IMG\" \\\n" + + " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &\n" + + "\n" + + "PID=$!\n" + + "\n" + + "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\n" + + "\n" + + "PID_SOCAT=$!\n" + + "\n" + + "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\n" + + "\n" + + "wait $PID") + .put("job", job); + + String token = slurmRest.generateRootToken("compute1"); + Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println); + } + + @Test + public void cancelJob() throws IOException, InterruptedException { + String token = slurmRest.generateRootToken("compute1"); + assert slurmRest.cancelJob(token, "5"); + + } + + @Test + public void runParlbert() throws IOException, InterruptedException { + JSONObject job = new JSONObject() + .put("name", "cuda_test") + .put("partition", "normal") + .put("time_limit", 600) + .put("required_nodes", "compute1") + .put("cpus_per_task", 2) + .put("current_working_directory", "/data") + .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/")); + + JSONObject payload = new JSONObject() + .put("script", + "#!/bin/bash\n" + + "#SBATCH --gres=gpu:1\n" + + "#SBATCH --partition=normal\n" + + "#SBATCH --nodelist=compute1\n" + + + "PORT=20000 \n" + + "INNER=9714 \n" + + "IMG=\"/sif_pool/bert_cuda.sif\" " + + "INTOIMAGE=\"cd /usr/src/app\"\n" + + "UVI=\"uvicorn parlbert_topic_german:app\" \n" + + "\n" + + "apptainer exec --nv \"$IMG\" \\\n" + + " sh -c \"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\" &\n" + + "\n" + + "PID=$!\n" + + "\n" + + "socat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\n" + + "\n" + + "PID_SOCAT=$!\n" + + "\n" + + "trap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\n" + + "\n" + + "wait $PID") + .put("job", job); + System.out.println(payload.toString()); + //String token = slurmRest.generateRootToken("compute1"); + //Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println); + } + + @Test + public void runNvidiasmi() throws IOException, InterruptedException { + JSONObject job = new JSONObject() + .put("name", "smi_test") + .put("partition", "normal") + .put("time_limit", 600) + .put("current_working_directory", "/data") + .put("environment", new JSONArray().put("PATH=/bin/:/usr/bin/:/sbin/")); + + JSONObject payload = new JSONObject() + .put("script", + "#!/bin/bash\n#SBATCH --gres=gpu:1\nnvidia-smi") + .put("job", job); + + String token = slurmRest.generateRootToken("compute1"); + Arrays.stream(slurmRest.submit(token, payload)).forEach(System.out::println); + } + + + @Test + public void pipelineTest_json() throws Exception { + SlurmRest rest = new SlurmRest("http://localhost"); + String token = rest.generateRootToken("compute1"); + + int iWorkers = 1; // + JCas jc = JCasFactory.createText("Early life\n" + + "William Gordon Gordon-Cumming was born on 20 July 1848 at Sanquhar House, near Forres, Morayshire.[1] His parents were Alexander Penrose Gordon-Cumming—the third of the Gordon-Cumming baronets—and Anne Pitcairn (née Campbell). William was the second of the couple's four children and their eldest son. His uncle, Roualeyn George Gordon-Cumming, was a noted big-game hunter; and his aunt, Constance Gordon-Cumming, was a travel writer. Gordon-Cumming was educated at the English boarding schools Eton and Wellington.[1][2]\n" + + "\n" + + "At the age of eighteen he inherited the baronetcy and became chief of Clan Cumming; his line had been traced from the fourth century, through Charlemagne. His inheritance included three Morayshire estates: Altyre near Forres, Gordonstoun near Elgin, and the village of Dallas. The estates totalled 38,500 acres (156 km2) of poor quality land;[1][3] the annual income from the estates in around 1890 has been given as either £60,000[4] or £80,000.[5][a]\n" + + "\n" + + "Military career\n" + + "Although Gordon-Cumming had asthma and was blind in one eye, he purchased a commission as ensign in the Scots Fusilier Guards (renamed in 1877 as the Scots Guards) in 1868 (dated from 25 December 1867).[1][7][8] He was promoted to regimental lieutenant and to the brevet rank of captain in the army by purchase on 17 May 1871, the last year in which commissions were available for purchase.[9][b] He volunteered for service in South Africa in the Anglo-Zulu War, where he served gallantly and was mentioned in despatches; he was the first man to enter Cetshwayo's kraal after the Battle of Ulundi (1879). That year he conveyed the condolences of the army to the ex-Empress Eugénie on the death of her son, Napoléon, Prince Imperial.[1][2]\n" + + "\n" + + "Gordon-Cumming was promoted to the regimental rank of captain and the army rank of lieutenant-colonel on 28 July 1880.[11] He served in Egypt during the Anglo-Egyptian War (1882) and in the Sudan in the Mahdist War (1884–1885), the last of which was with the Guards Camel Regiment in the Desert Column.[2][4][c] He was promoted to the regimental rank of major on 23 May 1888.[13]\n" + + "\n" + + "He also found time for independent travel and adventure, stalking tigers on foot in India and hunting in the Rocky Mountains in the US;[1][4] in 1871 he published an account of his travels in India, Wild Men and Wild Beasts: Scenes in Camp and Jungle.[14] The work covers the best routes to travel to and from India and which animals are available for hunting in which season, as well as the equipment a hunter would need to take on an expedition.[15] He concluded his work with the following:\n" + + "\n" + + "The record of my doings might no doubt have been more acceptable to the general reader had it been more varied with matter other than mere slaughter, and had the tale of bloodshed been more frequently relieved by accounts of the geography, scenery, and natural history, human and bestial, of the country; but all these have been well described elsewhere, and by abler pens.[16]\n" + + "\n" + + "Royal baccarat scandal\n" + + "Main article: Royal baccarat scandal\n" + + "In September 1890 Arthur Wilson, the 52-year-old Hull-based owner of a shipping business, invited Gordon-Cumming, along with Edward, Prince of Wales, to a house party at Tranby Croft in the East Riding of Yorkshire;[17][18] Gordon-Cumming and the prince had been friends for over twenty years.[19] Among the other people present that weekend were Wilson's wife, Mary, their son, Stanley, their daughter, Ethel, and her husband, Edward Lycett Green, who was the son of Sir Edward Green, 1st Baronet, a local Conservative politician.[20] Several members of the prince's inner circle were also invited to stay, including Sir Christopher Sykes—the Conservative MP for Beverley—the equerry Tyrwhitt Wilson, Lord Coventry, Lord Edward Somerset, Captain Arthur Somerset—his cousin—and Lieutenant-General Owen Williams, along with their wives. Also accompanying the party was Lieutenant Berkeley Levett, a brother officer to Gordon-Cumming in the Scots Guards and a friend of the Wilson family.[21]\n" + + "\n" + + "During the evenings of the weekend, Edward insisted on playing baccarat, a game that was at the time illegal if gambling was involved;[22] many of the house joined in, including Gordon-Cumming, Levett and Stanley Wilson. The prince acted as the dealer.[23][d] On the first night of play, Stanley Wilson thought he saw Gordon-Cumming add two red £5 counters onto his stake after the hand had finished, but before the winnings had been paid, thus increasing the money paid to him by the bank—a method of cheating known in casinos as la poussette. He alerted Levett, sitting next to him, and both men thought they saw Gordon-Cumming repeat the act on the next hand.[24][25]\n" + + "\n" + + "After the second evening of play Lycett Green, Stanley Wilson and Arthur and Edward Somerset confronted Gordon-Cumming and accused him of cheating.[18][e] Gordon-Cumming insisted they had been mistaken, and explained that he played the coup de trois system of betting,[f] in which if he won a hand with a £5 stake, he would add his winnings to the stake, together with another £5, as the stake for the next hand.[28][g] Edward, after hearing from his advisors and the accusers, believed what they had told him.[29] In order to avoid a scandal involving the prince, Gordon-Cumming gave way to pressure from the attendant royal courtiers to sign a statement undertaking never to play cards again in return for a pledge that no-one present would speak of the incident to anyone else.[30][31]\n" + + "\n" + + "In consideration of the promise made by the gentlemen whose names are subscribed to preserve my silence with reference to an accusation which has been made in regard to my conduct at baccarat on the nights of Monday and Tuesday the 8th and 9th at Tranby Croft, I will on my part solemnly undertake never to play cards again as long as I live.\n" + + "\n" + + "— (Signed) W. Gordon-Cumming[32]"); + DocumentMetaData dmd = new DocumentMetaData(jc); + dmd.setDocumentId("test_rest"); + dmd.setDocumentTitle("test_REST"); + dmd.addToIndexes(); + DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); + DUUILuaSandbox sandbox = new DUUILuaSandbox(); + sandbox._allowAllJavaClasses = true; + ctx.withSandbox(sandbox); + DUUIComposer composer = new DUUIComposer() + .withSkipVerification(true) + .withLuaContext(ctx) + .withWorkers(iWorkers); + DUUIUIMADriver uimaDriver = new DUUIUIMADriver(); + DUUISlurmDriver slurmDriver = new DUUISlurmDriver(rest); + + composer.addDriver(uimaDriver, slurmDriver); + + composer.resetPipeline(); + DUUIPipelineComponent com1 = new DUUISlurmDriver.Component( + new DUUIPipelineComponent().withSlurmScript( + "{\"job\":{\"environment\":[\"PATH=/bin/:/usr/bin/:/sbin/\"],\"partition\":\"normal\",\"time_limit\":600,\"name\":\"spacy_test\",\"current_working_directory\":\"/data\"},\"script\":\"#!/bin/bash\\n \\nPORT=20000 \\nINNER=9714 \\nIMG=\\\"/sif_pool/spacy.sif\\\" INTOIMAGE=\\\"cd /usr/src/app\\\"\\nUVI=\\\"uvicorn textimager_duui_spacy:app\\\" \\n\\napptainer exec --nv \\\"$IMG\\\" \\\\\\n sh -c \\\"$INTOIMAGE && $UVI --host 0.0.0.0 --port $INNER\\\" &\\n\\nPID=$!\\n\\nsocat TCP-LISTEN:$PORT,reuseaddr,fork TCP:127.0.0.1:$INNER &\\n\\nPID_SOCAT=$!\\n\\ntrap 'kill $PID $PID_SOCAT 2>/dev/null' EXIT\\n\\nwait $PID\"}\n" + ).withSlurmHostPort("20000")).build(); + + System.out.println(com1.getSlurmScript()); + + composer.add(com1); + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1", + XmiWriter.PARAM_COMPRESSION, "GZIP")) + .build()); + + composer.run(jc); + + + } + + @Test + public void pipelineTest_parameters() throws Exception { + SlurmRest rest = new SlurmRest("http://localhost"); + + + int iWorkers = 2; // + JCas jc = JCasFactory.createText("Early life\n" + + "William Gordon Gordon-Cumming was born on 20 July 1848 at Sanquhar House, near Forres, Morayshire.[1] His parents were Alexander Penrose Gordon-Cumming—the third of the Gordon-Cumming baronets—and Anne Pitcairn (née Campbell). William was the second of the couple's four children and their eldest son. His uncle, Roualeyn George Gordon-Cumming, was a noted big-game hunter; and his aunt, Constance Gordon-Cumming, was a travel writer. Gordon-Cumming was educated at the English boarding schools Eton and Wellington.[1][2]\n" + + "\n" + + "At the age of eighteen he inherited the baronetcy and became chief of Clan Cumming; his line had been traced from the fourth century, through Charlemagne. His inheritance included three Morayshire estates: Altyre near Forres, Gordonstoun near Elgin, and the village of Dallas. The estates totalled 38,500 acres (156 km2) of poor quality land;[1][3] the annual income from the estates in around 1890 has been given as either £60,000[4] or £80,000.[5][a]\n" + + "\n" + + "Military career\n" + + "Although Gordon-Cumming had asthma and was blind in one eye, he purchased a commission as ensign in the Scots Fusilier Guards (renamed in 1877 as the Scots Guards) in 1868 (dated from 25 December 1867).[1][7][8] He was promoted to regimental lieutenant and to the brevet rank of captain in the army by purchase on 17 May 1871, the last year in which commissions were available for purchase.[9][b] He volunteered for service in South Africa in the Anglo-Zulu War, where he served gallantly and was mentioned in despatches; he was the first man to enter Cetshwayo's kraal after the Battle of Ulundi (1879). That year he conveyed the condolences of the army to the ex-Empress Eugénie on the death of her son, Napoléon, Prince Imperial.[1][2]\n" + + "\n" + + "Gordon-Cumming was promoted to the regimental rank of captain and the army rank of lieutenant-colonel on 28 July 1880.[11] He served in Egypt during the Anglo-Egyptian War (1882) and in the Sudan in the Mahdist War (1884–1885), the last of which was with the Guards Camel Regiment in the Desert Column.[2][4][c] He was promoted to the regimental rank of major on 23 May 1888.[13]\n" + + "\n" + + "He also found time for independent travel and adventure, stalking tigers on foot in India and hunting in the Rocky Mountains in the US;[1][4] in 1871 he published an account of his travels in India, Wild Men and Wild Beasts: Scenes in Camp and Jungle.[14] The work covers the best routes to travel to and from India and which animals are available for hunting in which season, as well as the equipment a hunter would need to take on an expedition.[15] He concluded his work with the following:\n" + + "\n" + + "The record of my doings might no doubt have been more acceptable to the general reader had it been more varied with matter other than mere slaughter, and had the tale of bloodshed been more frequently relieved by accounts of the geography, scenery, and natural history, human and bestial, of the country; but all these have been well described elsewhere, and by abler pens.[16]\n" + + "\n" + + "Royal baccarat scandal\n" + + "Main article: Royal baccarat scandal\n" + + "In September 1890 Arthur Wilson, the 52-year-old Hull-based owner of a shipping business, invited Gordon-Cumming, along with Edward, Prince of Wales, to a house party at Tranby Croft in the East Riding of Yorkshire;[17][18] Gordon-Cumming and the prince had been friends for over twenty years.[19] Among the other people present that weekend were Wilson's wife, Mary, their son, Stanley, their daughter, Ethel, and her husband, Edward Lycett Green, who was the son of Sir Edward Green, 1st Baronet, a local Conservative politician.[20] Several members of the prince's inner circle were also invited to stay, including Sir Christopher Sykes—the Conservative MP for Beverley—the equerry Tyrwhitt Wilson, Lord Coventry, Lord Edward Somerset, Captain Arthur Somerset—his cousin—and Lieutenant-General Owen Williams, along with their wives. Also accompanying the party was Lieutenant Berkeley Levett, a brother officer to Gordon-Cumming in the Scots Guards and a friend of the Wilson family.[21]\n" + + "\n" + + "During the evenings of the weekend, Edward insisted on playing baccarat, a game that was at the time illegal if gambling was involved;[22] many of the house joined in, including Gordon-Cumming, Levett and Stanley Wilson. The prince acted as the dealer.[23][d] On the first night of play, Stanley Wilson thought he saw Gordon-Cumming add two red £5 counters onto his stake after the hand had finished, but before the winnings had been paid, thus increasing the money paid to him by the bank—a method of cheating known in casinos as la poussette. He alerted Levett, sitting next to him, and both men thought they saw Gordon-Cumming repeat the act on the next hand.[24][25]\n" + + "\n" + + "After the second evening of play Lycett Green, Stanley Wilson and Arthur and Edward Somerset confronted Gordon-Cumming and accused him of cheating.[18][e] Gordon-Cumming insisted they had been mistaken, and explained that he played the coup de trois system of betting,[f] in which if he won a hand with a £5 stake, he would add his winnings to the stake, together with another £5, as the stake for the next hand.[28][g] Edward, after hearing from his advisors and the accusers, believed what they had told him.[29] In order to avoid a scandal involving the prince, Gordon-Cumming gave way to pressure from the attendant royal courtiers to sign a statement undertaking never to play cards again in return for a pledge that no-one present would speak of the incident to anyone else.[30][31]\n" + + "\n" + + "In consideration of the promise made by the gentlemen whose names are subscribed to preserve my silence with reference to an accusation which has been made in regard to my conduct at baccarat on the nights of Monday and Tuesday the 8th and 9th at Tranby Croft, I will on my part solemnly undertake never to play cards again as long as I live.\n" + + "\n" + + "— (Signed) W. Gordon-Cumming[32]"); + DocumentMetaData dmd = new DocumentMetaData(jc); + dmd.setDocumentId("test_rest"); + dmd.setDocumentTitle("test_REST"); + dmd.addToIndexes(); + DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); + DUUILuaSandbox sandbox = new DUUILuaSandbox(); + sandbox._allowAllJavaClasses = true; + ctx.withSandbox(sandbox); + DUUIComposer composer = new DUUIComposer() + .withSkipVerification(true) + .withLuaContext(ctx) + .withWorkers(iWorkers); + DUUIUIMADriver uimaDriver = new DUUIUIMADriver(); + DUUISlurmDriver slurmDriver = new DUUISlurmDriver(rest); + + composer.addDriver(uimaDriver, slurmDriver); + + composer.resetPipeline(); + DUUIPipelineComponent com1 = new DUUISlurmDriver.Component( + new DUUIPipelineComponent(). + withSlurmPartition("normal"). + withSlurmNodelist("compute1"). + withSlurmWorkDir("/data"). + withSlurmUvicorn("uvicorn textimager_duui_spacy:app"). + withSlurmSIFName("spacy"). + withSlurmMemory("1024"). + withSlurmRuntime("5").withSlurmCPUs("2"). + withSlurmSaveIn("/sif_pool/spacy.sif").withSlurmJobName("spacy").withSlurmCPUs("2"). + withSlurmGPU("0").withScale(2) + ).build(); + composer.add(com1); + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1", + XmiWriter.PARAM_COMPRESSION, "GZIP")) + .build()); + composer.run(jc); + + } + +} diff --git a/src/test/java/slurmDriver/universTest.java b/src/test/java/slurmDriver/universTest.java new file mode 100755 index 00000000..777b4c8c --- /dev/null +++ b/src/test/java/slurmDriver/universTest.java @@ -0,0 +1,96 @@ +package slurmDriver; + +import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; +import org.apache.uima.fit.factory.JCasFactory; +import org.apache.uima.jcas.JCas; +import org.dkpro.core.io.xmi.XmiWriter; +import org.junit.jupiter.api.Test; +import org.texttechnologylab.DockerUnifiedUIMAInterface.DUUIComposer; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIDockerDriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIPipelineComponent; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.DUUIUIMADriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.DUUISlurmDriver; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.SlurmUtils; +import org.texttechnologylab.DockerUnifiedUIMAInterface.driver.slurm.slurmInDocker.SlurmRest; +import org.texttechnologylab.DockerUnifiedUIMAInterface.lua.*; + +import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; + +public class universTest { + @Test + public void universTest() throws Exception { + int iWorkers = 2; // define the number of workers + + JCas jc = JCasFactory.createJCas(); // An empty CAS document is defined. + +// load content into jc ... + +// Defining LUA-Context for communication + DUUILuaContext ctx = LuaConsts.getJSON(); +// The composer is defined and initialized with a standard Lua context as well with a storage backend. + DUUIComposer composer = new DUUIComposer().withLuaContext(ctx).withSkipVerification(true); + + DUUIUIMADriver uima_driver = new DUUIUIMADriver(); + +// A driver must be added before components can be added for it in the composer. After that the composer is able to use the individual drivers. + composer.addDriver(uima_driver); +// Adding a UIMA annotator for writing the result of the pipeline as XMI files. + composer.add(new DUUIUIMADriver.Component( + createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/test" + )).withScale(iWorkers)); + +// The document is processed through the pipeline. In addition, files of entire repositories can be processed. + composer.run(jc); + } + + + @Test + public void universTest2() throws Exception { + + int iWorkers = 5; // define the number of workers + + JCas jc = JCasFactory.createText("hello world"); + DocumentMetaData dmd = new DocumentMetaData(jc); + dmd.setDocumentId("test01"); + dmd.setDocumentTitle("univers test2"); + dmd.addToIndexes(); + + + // + + + DUUILuaContext ctx = new DUUILuaContext().withJsonLibrary(); + DUUILuaSandbox sandbox = new DUUILuaSandbox(); + sandbox._allowAllJavaClasses = true; + ctx.withSandbox(sandbox); + DUUIComposer composer = new DUUIComposer() + .withSkipVerification(true) + .withLuaContext(ctx) + .withWorkers(iWorkers); + + DUUIUIMADriver uimaDriver = new DUUIUIMADriver(); + DUUIDockerDriver duuidDriver = new DUUIDockerDriver(); + composer.addDriver(uimaDriver, duuidDriver); + // reset also init make sure before run empty + composer.resetPipeline(); + + + composer.add(new DUUIDockerDriver.Component("docker.texttechnologylab.org/textimager-duui-spacy-single-de_core_news_sm:0.1.4") + .withImageFetching() + .withScale(iWorkers) + .build()); + + composer.add(new DUUIUIMADriver.Component(createEngineDescription(XmiWriter.class, + XmiWriter.PARAM_TARGET_LOCATION, "/tmp/nlp/", + XmiWriter.PARAM_PRETTY_PRINT, true, + XmiWriter.PARAM_OVERWRITE, true, + XmiWriter.PARAM_VERSION, "1.1", + XmiWriter.PARAM_COMPRESSION, "GZIP")) + .build()); + + composer.run(jc); + } + + +}