From 945cdbdbe7bf143262e895ca18d1c49955574c86 Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Thu, 22 Jan 2026 10:09:36 +0100 Subject: [PATCH 1/3] Resolve JS module on first use if empty --- .../indexdata/reservoir/module/Module.java | 2 +- .../module/impl/ModuleCacheImpl.java | 4 +- .../module/impl/ModuleJavaScript.java | 48 ++++++++++++------- .../reservoir/module/impl/ModuleJsonPath.java | 2 +- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/com/indexdata/reservoir/module/Module.java b/server/src/main/java/com/indexdata/reservoir/module/Module.java index 17bf42b4..9a34edcb 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/Module.java +++ b/server/src/main/java/com/indexdata/reservoir/module/Module.java @@ -8,7 +8,7 @@ public interface Module { - Future initialize(Vertx vertx, CodeModuleEntity entity); + Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity); Future execute(String symbol, JsonObject input); diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java index c13fd593..c4d98b26 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java @@ -61,8 +61,8 @@ public Future lookup(Vertx vertx, String tenantId, CodeModuleEntity enti entries.remove(cacheKey); } Module module = createInstance(entity.getType()); - return module.initialize(vertx, entity).map(x -> { - CacheEntry e = new CacheEntry(module, entity); + return module.initialize(vertx, tenantId, entity).map(newEntity -> { + CacheEntry e = new CacheEntry(module, newEntity); entries.put(cacheKey, e); return module; }); diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java index f72fc8ad..0036052a 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java @@ -1,9 +1,11 @@ package com.indexdata.reservoir.module.impl; import com.indexdata.reservoir.module.Module; +import com.indexdata.reservoir.server.Storage; import com.indexdata.reservoir.server.entity.CodeModuleEntity; import io.vertx.core.Future; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; import io.vertx.core.json.JsonObject; import java.util.Collection; import java.util.HashSet; @@ -22,7 +24,7 @@ public class ModuleJavaScript implements Module { private Vertx vertx; @Override - public Future initialize(Vertx vertx, CodeModuleEntity entity) { + public Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity) { id = entity.getId(); if (id == null || id.isEmpty()) { return Future.failedFuture( @@ -31,29 +33,43 @@ public Future initialize(Vertx vertx, CodeModuleEntity entity) { this.vertx = vertx; String url = entity.getUrl(); String script = entity.getScript(); - if (script == null || script.isEmpty()) { + var hasUrl = url != null && !url.isEmpty(); + var hasScript = script != null && !script.isEmpty(); + if (!hasUrl && !hasScript) { return Future.failedFuture( new IllegalArgumentException("Module config must include 'url' or 'script'")); } - if (url != null && !url.isEmpty()) { - // url always points to an ES module - defaultFunctionName = entity.getFunction(); - final boolean isModule = url.endsWith("mjs"); - if (!isModule) { - return Future.failedFuture(new IllegalArgumentException( - "url must end with .mjs to designate ES module")); + if (hasUrl) { + //module was never resolved (migration) + if (!hasScript) { + return new CodeModuleEntity.CodeModuleBuilder(entity.asJson()) + .resolve(vertx) + .compose(newEntity -> new Storage(vertx, tenant, HttpMethod.POST) + .updateCodeModuleEntity(newEntity).map(newEntity)) + .compose(this::initAsEsModule); } - Context.Builder cb = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("js.esm-eval-returns-exports", "true"); - context = cb.build(); - String moduleName = url.substring(url.lastIndexOf("/") + 1); - module = context.eval(Source.newBuilder("js", script, moduleName).buildLiteral()); + return this.initAsEsModule(entity); } else { context = Context.create("js"); function = context.eval("js", script); + return Future.succeededFuture(entity); } - return Future.succeededFuture(); + } + + private Future initAsEsModule(CodeModuleEntity entity) { + defaultFunctionName = entity.getFunction(); + final boolean isModule = entity.getUrl().endsWith("mjs"); + if (!isModule) { + return Future.failedFuture(new IllegalArgumentException( + "url must end with .mjs to designate ES module")); + } + Context.Builder cb = Context.newBuilder("js") + .allowExperimentalOptions(true) + .option("js.esm-eval-returns-exports", "true"); + context = cb.build(); + String moduleName = entity.getUrl().substring(entity.getUrl().lastIndexOf("/") + 1); + module = context.eval(Source.newBuilder("js", entity.getScript(), moduleName).buildLiteral()); + return Future.succeededFuture(entity); } private Value getFunction(String functionName) { diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java index fdaf597d..c077c65e 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java @@ -17,7 +17,7 @@ public class ModuleJsonPath implements Module { JsonPath jsonPath; @Override - public Future initialize(Vertx vertx, CodeModuleEntity entity) { + public Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity) { String script = entity.getScript(); if (script == null) { return Future.failedFuture("module config must include 'script'"); From 80cece72e85fa908848f881c230f10c2ea827e5d Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Thu, 22 Jan 2026 10:31:34 +0100 Subject: [PATCH 2/3] Fix NPE --- .../com/indexdata/reservoir/module/impl/ModuleJsonPath.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java index c077c65e..a1e9a0f2 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java @@ -23,7 +23,7 @@ public Future initialize(Vertx vertx, String tenant, CodeModul return Future.failedFuture("module config must include 'script'"); } jsonPath = JsonPath.compile(script); - return Future.succeededFuture(); + return Future.succeededFuture(entity); } public ModuleJsonPath() { From 4737b6cf872556c3f9eaf6bdb692957dd1e62098 Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Thu, 22 Jan 2026 11:09:27 +0100 Subject: [PATCH 3/3] Move update to Cache impl --- .../indexdata/reservoir/module/Module.java | 2 +- .../reservoir/module/ModuleCache.java | 3 +++ .../module/impl/ModuleCacheImpl.java | 26 ++++++++++++++++--- .../module/impl/ModuleJavaScript.java | 4 +-- .../reservoir/module/impl/ModuleJsonPath.java | 2 +- .../reservoir/server/ReservoirService.java | 6 ++--- .../indexdata/reservoir/server/Storage.java | 4 +-- 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/indexdata/reservoir/module/Module.java b/server/src/main/java/com/indexdata/reservoir/module/Module.java index 9a34edcb..e0ff153e 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/Module.java +++ b/server/src/main/java/com/indexdata/reservoir/module/Module.java @@ -8,7 +8,7 @@ public interface Module { - Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity); + Future initialize(Vertx vertx, CodeModuleEntity entity); Future execute(String symbol, JsonObject input); diff --git a/server/src/main/java/com/indexdata/reservoir/module/ModuleCache.java b/server/src/main/java/com/indexdata/reservoir/module/ModuleCache.java index 91812ada..82fa1bfd 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/ModuleCache.java +++ b/server/src/main/java/com/indexdata/reservoir/module/ModuleCache.java @@ -1,6 +1,7 @@ package com.indexdata.reservoir.module; import com.indexdata.reservoir.module.impl.ModuleCacheImpl; +import com.indexdata.reservoir.server.Storage; import com.indexdata.reservoir.server.entity.CodeModuleEntity; import io.vertx.core.Future; import io.vertx.core.Vertx; @@ -11,6 +12,8 @@ static ModuleCache getInstance() { return ModuleCacheImpl.getInstance(); } + public Future lookup(Vertx vertx, Storage storage, CodeModuleEntity entity); + public Future lookup(Vertx vertx, String tenant, CodeModuleEntity entity); void purge(String tenant, String id); diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java index c4d98b26..b2b3ed66 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleCacheImpl.java @@ -2,9 +2,11 @@ import com.indexdata.reservoir.module.Module; import com.indexdata.reservoir.module.ModuleCache; +import com.indexdata.reservoir.server.Storage; import com.indexdata.reservoir.server.entity.CodeModuleEntity; import io.vertx.core.Future; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; import java.util.HashMap; import java.util.Map; @@ -46,12 +48,22 @@ private Module createInstance(String type) { } @Override - public Future lookup(Vertx vertx, String tenantId, CodeModuleEntity entity) { + public Future lookup(Vertx vertx, String tenant, CodeModuleEntity entity) { + return lookup(vertx, tenant, null, entity); + } + + @Override + public Future lookup(Vertx vertx, Storage storage, CodeModuleEntity entity) { + return lookup(vertx, storage.getTenant(), storage, entity); + } + + private Future lookup(Vertx vertx, String tenant, Storage storage, + CodeModuleEntity entity) { String moduleId = entity.getId(); if (moduleId == null) { return Future.failedFuture("module config must include 'id'"); } - String cacheKey = tenantId + ":" + moduleId; + String cacheKey = tenant + ":" + moduleId; CacheEntry entry = entries.get(cacheKey); if (entry != null) { if (entry.entity.equals(entity)) { @@ -61,7 +73,15 @@ public Future lookup(Vertx vertx, String tenantId, CodeModuleEntity enti entries.remove(cacheKey); } Module module = createInstance(entity.getType()); - return module.initialize(vertx, tenantId, entity).map(newEntity -> { + return module.initialize(vertx, entity) + .compose(newEntity -> { + if (storage != null && newEntity != entity) { + return storage.updateCodeModuleEntity(newEntity).map(newEntity); + } else { + return Future.succeededFuture(newEntity); + } + }) + .map(newEntity -> { CacheEntry e = new CacheEntry(module, newEntity); entries.put(cacheKey, e); return module; diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java index 0036052a..5d6544ac 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJavaScript.java @@ -24,7 +24,7 @@ public class ModuleJavaScript implements Module { private Vertx vertx; @Override - public Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity) { + public Future initialize(Vertx vertx, CodeModuleEntity entity) { id = entity.getId(); if (id == null || id.isEmpty()) { return Future.failedFuture( @@ -44,8 +44,6 @@ public Future initialize(Vertx vertx, String tenant, CodeModul if (!hasScript) { return new CodeModuleEntity.CodeModuleBuilder(entity.asJson()) .resolve(vertx) - .compose(newEntity -> new Storage(vertx, tenant, HttpMethod.POST) - .updateCodeModuleEntity(newEntity).map(newEntity)) .compose(this::initAsEsModule); } return this.initAsEsModule(entity); diff --git a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java index a1e9a0f2..2ab33494 100644 --- a/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java +++ b/server/src/main/java/com/indexdata/reservoir/module/impl/ModuleJsonPath.java @@ -17,7 +17,7 @@ public class ModuleJsonPath implements Module { JsonPath jsonPath; @Override - public Future initialize(Vertx vertx, String tenant, CodeModuleEntity entity) { + public Future initialize(Vertx vertx, CodeModuleEntity entity) { String script = entity.getScript(); if (script == null) { return Future.failedFuture("module config must include 'script'"); diff --git a/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java b/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java index bba92ecd..ea83a11e 100644 --- a/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java +++ b/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java @@ -98,7 +98,7 @@ Future reloadCodeModule(RoutingContext ctx) { } return new CodeModuleEntity.CodeModuleBuilder(res.asJson()) .resolve(ctx.vertx()) - .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), tenant, cm) + .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), storage, cm) .compose(module -> storage.updateCodeModuleEntity(cm)) .compose(x -> ctx.response().setStatusCode(204).end()) ); @@ -390,7 +390,7 @@ Future postCodeModule(RoutingContext ctx) { JsonObject request = validatedRequest.getBody().getJsonObject(); return new CodeModuleEntity.CodeModuleBuilder(request) .resolve(ctx.vertx()) - .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), TenantUtil.tenant(ctx), cm) + .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), storage, cm) .compose(module -> storage.insertCodeModuleEntity(cm)) .compose(res -> HttpResponse.responseJson(ctx, 201) @@ -421,7 +421,7 @@ Future putCodeModule(RoutingContext ctx) { JsonObject request = validatedRequest.getBody().getJsonObject(); return new CodeModuleEntity.CodeModuleBuilder(request) .resolve(ctx.vertx()) - .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), TenantUtil.tenant(ctx), cm) + .compose(cm -> ModuleCache.getInstance().lookup(ctx.vertx(), storage, cm) .compose(module -> storage.updateCodeModuleEntity(cm)) .compose(res -> { if (Boolean.FALSE.equals(res)) { diff --git a/server/src/main/java/com/indexdata/reservoir/server/Storage.java b/server/src/main/java/com/indexdata/reservoir/server/Storage.java index ce1b3f7e..e3b9142c 100644 --- a/server/src/main/java/com/indexdata/reservoir/server/Storage.java +++ b/server/src/main/java/com/indexdata/reservoir/server/Storage.java @@ -392,7 +392,7 @@ Future createIngestMatcher(JsonObject matchKeyConfig, Vertx vertx "Module '" + invocation.getModuleName() + "' does not exist for '" + invocation + "'"); } - return ModuleCache.getInstance().lookup(vertx, tenant, entity) + return ModuleCache.getInstance().lookup(vertx, this, entity) .compose(module -> { ingestMatcher.moduleExecutable = new ModuleExecutable(module, invocation); return Future.succeededFuture(ingestMatcher); @@ -1327,7 +1327,7 @@ Future getTransformer(RoutingContext ctx) { return Future.failedFuture("Transformer module '" + invocation.getModuleName() + "' not found"); } - return ModuleCache.getInstance().lookup(ctx.vertx(), TenantUtil.tenant(ctx), entity) + return ModuleCache.getInstance().lookup(ctx.vertx(), this, entity) .map(mod -> new ModuleExecutable(mod, invocation)); }); });