diff --git a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethod.java b/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethod.java deleted file mode 100644 index c9d460a3..00000000 --- a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethod.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.indexdata.reservoir.matchkey; - -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; -import java.util.Collection; - -/** - * Don't use this. - * @deprecated for removal - */ -@Deprecated(forRemoval = true, since = "1.0") -public interface MatchKeyMethod { - - /** - * Get MatchKeyMethod instance with configuration. - * @param vertx Vert.x handle - * @param tenant tenant - * @param id matchkey id - * @param method method name - * @param configuration configuration - * @return Async result MatchKeyMethod - */ - static Future get(Vertx vertx, String tenant, String id, - String method, JsonObject configuration) { - return MatchKeyMethodFactory.get(vertx, tenant, id, method, configuration); - } - - Future configure(Vertx vertx, JsonObject configuration); - - /** - * Generate match keys. - * @param payload payload with marc and inventory XSLT result - * @param keys resulting keys (unmodified if no keys were generated). - */ - void getKeys(JsonObject payload, Collection keys); - - /** - * Close resources for method. - */ - default void close() { } -} diff --git a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodEntry.java b/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodEntry.java deleted file mode 100644 index aa8bd1e8..00000000 --- a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodEntry.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.indexdata.reservoir.matchkey; - -import io.vertx.core.json.JsonObject; - -/** - * Don't use this. - * @deprecated for removal - */ -@Deprecated(forRemoval = true, since = "1.0") -public class MatchKeyMethodEntry { - JsonObject conf; - MatchKeyMethod method; -} diff --git a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactory.java b/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactory.java deleted file mode 100644 index c9012e0f..00000000 --- a/server/src/main/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactory.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.indexdata.reservoir.matchkey; - -import com.indexdata.reservoir.matchkey.impl.MatchKeyJavaScript; -import com.indexdata.reservoir.matchkey.impl.MatchKeyJsonPath; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; -import java.util.HashMap; -import java.util.Map; - -/** - * Don't use this. - * @deprecated for removal - */ -@Deprecated(forRemoval = true, since = "1.0") -public final class MatchKeyMethodFactory { - - private MatchKeyMethodFactory() { - throw new UnsupportedOperationException("MatchKeyMethodFactory"); - } - - private static Map instances = new HashMap<>(); - - /** - * Get MatchKeyMethod instance from method. - * @param method method name - * @return method or NULL if not found - */ - public static MatchKeyMethod get(String method) { - if ("jsonpath".equals(method)) { - return new MatchKeyJsonPath(); - } else if ("javascript".equals(method)) { - return new MatchKeyJavaScript(); - } - return null; - } - - static Future get(Vertx vertx, String tenant, String id, - String method, JsonObject configuration) { - String primaryKey = tenant + "-" + id; - JsonObject conf = new JsonObject() - .put("method", method) - .put("params", configuration); - - MatchKeyMethodEntry entry = instances.get(primaryKey); - if (entry != null) { - if (entry.conf.equals(conf)) { - return Future.succeededFuture(entry.method); - } - entry.method.close(); - instances.remove(primaryKey); - } - MatchKeyMethod m = get(method); - if (m == null) { - return Future.failedFuture("Unknown match key method " + method); - } - try { - return m.configure(vertx, configuration).map(x -> { - MatchKeyMethodEntry newEntry = new MatchKeyMethodEntry(); - newEntry.conf = conf; - newEntry.method = m; - instances.put(primaryKey, newEntry); - return m; - }); - } catch (Exception e) { - return Future.failedFuture(e); - } - } - - static void clearCache() { - instances.forEach((x,y) -> y.method.close()); - instances.clear(); - } - - static int getCacheSize() { - return instances.size(); - } -} diff --git a/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScript.java b/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScript.java deleted file mode 100644 index 5ba4ed15..00000000 --- a/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScript.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.indexdata.reservoir.matchkey.impl; - -import com.indexdata.reservoir.matchkey.MatchKeyMethod; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpResponseExpectation; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.client.WebClient; -import java.util.Collection; -import org.folio.okapi.common.WebClientFactory; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.Source; -import org.graalvm.polyglot.Value; - -/** - * Don't use this. - * @deprecated for removal - */ -@Deprecated(forRemoval = true, since = "1.0") -public class MatchKeyJavaScript implements MatchKeyMethod { - - Value getKeysFunction; - Context context; - - Future evalUrl(Vertx vertx, String url) { - WebClient webClient = WebClientFactory.getWebClient(vertx); - String moduleName = url.substring(url.lastIndexOf("/") + 1); - return webClient.getAbs(url) - .send() - .expecting(HttpResponseExpectation.SC_OK) - .map(response -> context.eval(Source - .newBuilder("js", response.bodyAsString(), moduleName) - .buildLiteral())); - } - - @Override - public Future configure(Vertx vertx, JsonObject configuration) { - String url = configuration.getString("url"); - String script = configuration.getString("script"); - if (url == null && script == null) { - return Future.failedFuture( - new IllegalArgumentException("javascript: url or script must be given")); - } - Future future = Future.succeededFuture(); - if (url != null) { - // if url is specified and ends with mjs, assume it is a ES module that exports a - // 'matchkey' function, otherwise treat it like a regular script - final boolean isModule = url.endsWith("mjs"); - if (isModule) { - context = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("js.esm-eval-returns-exports", "true") - .build(); - } else { - context = Context.create("js"); - } - future = evalUrl(vertx, url) - .map(value -> getKeysFunction = isModule ? value.getMember("matchkey") : value) - .mapEmpty(); - } - // if script is specified, we treat it as a regular, non-module JS file which - // evaluates to a function that accepts an object and returns an array of strings - if (script != null) { - if (context == null) { - context = Context.create("js"); - } - future = future - .map(v -> getKeysFunction = context.eval("js", script)) - .mapEmpty(); - } - return future; - - } - - private void addValue(Collection keys, Value value) { - if (value.isNumber()) { - keys.add(Long.toString(value.asLong())); - } else if (value.isString()) { - keys.add(value.asString()); - } - } - - @Override - public void getKeys(JsonObject payload, Collection keys) { - Value value = getKeysFunction.execute(payload.encode()); - if (value.hasArrayElements()) { - for (int i = 0; i < value.getArraySize(); i++) { - Value memberValue = value.getArrayElement(i); - addValue(keys, memberValue); - } - } else { - addValue(keys, value); - } - } - - @Override - public void close() { - if (context != null) { - context.close(true); - context = null; - } - } -} diff --git a/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPath.java b/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPath.java deleted file mode 100644 index 3cf6cc28..00000000 --- a/server/src/main/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPath.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.indexdata.reservoir.matchkey.impl; - -import com.indexdata.reservoir.matchkey.MatchKeyMethod; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.ReadContext; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; -import java.util.Collection; -import java.util.List; - -/** - * Don't use this. - * @deprecated for removal - */ -@Deprecated(forRemoval = true, since = "1.0") -public class MatchKeyJsonPath implements MatchKeyMethod { - - JsonPath jsonPath; - - @Override - public Future configure(Vertx vertx, JsonObject configuration) { - String expr = configuration.getString("expr"); - if (expr == null) { - return Future.failedFuture("jsonpath: expr must be given"); - } - jsonPath = JsonPath.compile(expr); - return Future.succeededFuture(); - } - - @Override - public void getKeys(JsonObject payload, Collection keys) { - ReadContext ctx = JsonPath.parse(payload.encode()); - try { - Object o = ctx.read(jsonPath); - if (o instanceof String string) { - keys.add(string); - } else if (o instanceof List list) { - for (Object m : list) { - if (!(m instanceof String)) { - return; - } - } - keys.addAll((List) o); - } - } catch (PathNotFoundException e) { - // ignored.. no keys added - } - } - -} diff --git a/server/src/main/java/com/indexdata/reservoir/server/IngestMatcher.java b/server/src/main/java/com/indexdata/reservoir/server/IngestMatcher.java index 88c3bfea..19069226 100644 --- a/server/src/main/java/com/indexdata/reservoir/server/IngestMatcher.java +++ b/server/src/main/java/com/indexdata/reservoir/server/IngestMatcher.java @@ -1,10 +1,8 @@ package com.indexdata.reservoir.server; -import com.indexdata.reservoir.matchkey.MatchKeyMethod; import com.indexdata.reservoir.module.ModuleExecutable; public class IngestMatcher { String matchKeyId; ModuleExecutable moduleExecutable; - MatchKeyMethod matchKeyMethod; } diff --git a/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java b/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java index bba92ecd..10701afc 100644 --- a/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java +++ b/server/src/main/java/com/indexdata/reservoir/server/ReservoirService.java @@ -1,6 +1,5 @@ package com.indexdata.reservoir.server; -import com.indexdata.reservoir.matchkey.MatchKeyMethodFactory; import com.indexdata.reservoir.module.ModuleCache; import com.indexdata.reservoir.module.ModuleInvocation; import com.indexdata.reservoir.server.entity.CodeModuleEntity; @@ -250,9 +249,6 @@ Future getCluster(RoutingContext ctx) { static String getMethod(JsonObject config) { String method = config.getString("method"); - if (method != null && MatchKeyMethodFactory.get(method) == null) { - throw new IllegalArgumentException("Non-existing method '" + method + "'"); - } return method; } @@ -395,7 +391,7 @@ Future postCodeModule(RoutingContext ctx) { .compose(res -> HttpResponse.responseJson(ctx, 201) .putHeader("Location", ctx.request().absoluteURI() + "/" + cm.getId()) - .end(cm.asJson().encode()) + .end(cm.asJson(true).encode()) ) ); } @@ -410,7 +406,7 @@ Future getCodeModule(RoutingContext ctx) { String.format(ENTITY_ID_NOT_FOUND_PATTERN, MODULE_LABEL, id)); return; } - HttpResponse.responseJson(ctx, 200).end(e.asJson().encode()); + HttpResponse.responseJson(ctx, 200).end(e.asJson(true).encode()); }) .mapEmpty(); } diff --git a/server/src/main/java/com/indexdata/reservoir/server/Storage.java b/server/src/main/java/com/indexdata/reservoir/server/Storage.java index 04450bd8..d445a15f 100644 --- a/server/src/main/java/com/indexdata/reservoir/server/Storage.java +++ b/server/src/main/java/com/indexdata/reservoir/server/Storage.java @@ -1,6 +1,5 @@ package com.indexdata.reservoir.server; -import com.indexdata.reservoir.matchkey.MatchKeyMethod; import com.indexdata.reservoir.module.ModuleCache; import com.indexdata.reservoir.module.ModuleExecutable; import com.indexdata.reservoir.module.ModuleInvocation; @@ -362,14 +361,6 @@ Future runMatcher(IngestMatcher ingestMatcher, IngestMetrics inge return result; }); } - if (ingestMatcher.matchKeyMethod != null) { - HashSet values = new HashSet<>(); - ingestMatcher.matchKeyMethod.getKeys(payload, values); - values.forEach(k -> result.keys.add(k.length() > MATCHVALUE_MAX_LENGTH - ? k.substring(0, MATCHVALUE_MAX_LENGTH) : k)); - - ingestMetrics.recordMatcher(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); - } return Future.succeededFuture(result); } @@ -403,16 +394,7 @@ Future createIngestMatcher(JsonObject matchKeyConfig, Vertx vertx }); }); } - String methodName = matchKeyConfig.getString("method"); - if (methodName != null) { - JsonObject params = matchKeyConfig.getJsonObject("params"); - return MatchKeyMethod.get(vertx, tenant, ingestMatcher.matchKeyId, methodName, params) - .compose(matchKeyMethod -> { - ingestMatcher.matchKeyMethod = matchKeyMethod; - return Future.succeededFuture(ingestMatcher); - }); - } - return Future.failedFuture("match key config must include 'method' or 'matcher'"); + return Future.failedFuture("match key config must include 'matcher'"); } Future> createIngestMatchers(JsonArray matchKeyConfigs, Vertx vertx) { diff --git a/server/src/main/resources/openapi/schemas/matchKey.json b/server/src/main/resources/openapi/schemas/matchKey.json index 625e0ce0..f03c4a44 100644 --- a/server/src/main/resources/openapi/schemas/matchKey.json +++ b/server/src/main/resources/openapi/schemas/matchKey.json @@ -10,16 +10,6 @@ "type": "string", "description": "matcher module invocation: 'moduleId<::function>' where <::function> is optional" }, - "method": { - "type": "string", - "description": "DEPRECATED: use 'matcher' instead", - "deprecated": true - }, - "params": { - "type": "object", - "description": "DEPRECATED: use 'matcher' instead", - "deprecated": true - }, "update": { "type": "string", "enum": ["ingest", "manual"], diff --git a/server/src/test/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactoryTest.java b/server/src/test/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactoryTest.java deleted file mode 100644 index 26e2ecc8..00000000 --- a/server/src/test/java/com/indexdata/reservoir/matchkey/MatchKeyMethodFactoryTest.java +++ /dev/null @@ -1,109 +0,0 @@ -package com.indexdata.reservoir.matchkey; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; - -import com.jayway.jsonpath.InvalidPathException; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.unit.TestContext; -import io.vertx.ext.unit.junit.VertxUnitRunner; -import org.folio.okapi.testing.UtilityClassTester; -import org.hamcrest.Matchers; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(VertxUnitRunner.class) -public class MatchKeyMethodFactoryTest { - static final String TENANT = "tenant"; - - static final String MATCHKEYID = "matchkeyid"; - static Vertx vertx; - - @BeforeClass - public static void beforeClass() { - vertx = Vertx.vertx(); - } - - @AfterClass - public static void afterClass(TestContext context) { - vertx.close().onComplete(context.asyncAssertSuccess()); - } - - @Test - public void isUtilityClass() { - UtilityClassTester.assertUtilityClass(MatchKeyMethodFactory.class); - } - - @Test - public void matchKeyJsonPathNonConfigured(TestContext context) { - MatchKeyMethodFactory.get(vertx, TENANT, MATCHKEYID, "foo", new JsonObject()).onComplete(context.asyncAssertFailure(e -> - assertThat(e.getMessage(), Matchers.is("Unknown match key method foo")) - )); - } - - @Test - public void testSameConfig(TestContext context) { - MatchKeyMethodFactory.clearCache(); - MatchKeyMethodFactory.get(vertx, TENANT, MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")).compose(m1 -> - MatchKeyMethod.get(vertx, TENANT, MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")) - .onComplete(context.asyncAssertSuccess(m2 -> { - assertThat(m1, is(m2)); - assertThat(MatchKeyMethodFactory.getCacheSize(), is(1)); - }))); - } - - @Test - public void testDiffConfig(TestContext context) { - MatchKeyMethodFactory.clearCache(); - MatchKeyMethodFactory.get(vertx, TENANT, MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")).compose(m1 -> - MatchKeyMethodFactory.get(vertx, TENANT, MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'y'")) - .onComplete(context.asyncAssertSuccess(m2 -> { - assertThat(m1, not(m2)); - assertThat(MatchKeyMethodFactory.getCacheSize(), is(1)); - }))); - } - - @Test - public void testDiffKey(TestContext context) { - MatchKeyMethodFactory.clearCache(); - MatchKeyMethodFactory.get(vertx, TENANT, "key1", "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")).compose(m1 -> - MatchKeyMethodFactory.get(vertx, TENANT, "key2", "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")) - .onComplete(context.asyncAssertSuccess(m2 -> { - assertThat(m1, not(m2)); - assertThat(MatchKeyMethodFactory.getCacheSize(), is(2)); - }))); - } - - @Test - public void testDiffTenant(TestContext context) { - MatchKeyMethodFactory.clearCache(); - MatchKeyMethodFactory.get(vertx, "t1", MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")).compose(m1 -> - MatchKeyMethod.get(vertx, "t2", MATCHKEYID, "javascript", new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")) - .onComplete(context.asyncAssertSuccess(m2 -> { - assertThat(m1, not(m2)); - assertThat(MatchKeyMethodFactory.getCacheSize(), is(2)); - }))); - } - - @Test - public void matchKeyJsonPathConfigureInvalidJsonPath(TestContext context) { - JsonObject configuration = new JsonObject().put("expr", "$.fields.010.subfields[x"); - MatchKeyMethodFactory.get(vertx, TENANT, MATCHKEYID, "jsonpath", configuration) - .onComplete(context.asyncAssertFailure(e -> - assertThat(e.getClass(), is(InvalidPathException.class)) - )); - } - -} diff --git a/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScriptTest.java b/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScriptTest.java deleted file mode 100644 index b8c6ce8e..00000000 --- a/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJavaScriptTest.java +++ /dev/null @@ -1,232 +0,0 @@ -package com.indexdata.reservoir.matchkey.impl; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; - -import com.indexdata.reservoir.matchkey.MatchKeyMethod; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerResponse; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.unit.TestContext; -import io.vertx.ext.unit.junit.VertxUnitRunner; -import io.vertx.ext.web.Router; -import io.vertx.reactivex.core.http.HttpHeaders; -import java.util.Collection; -import java.util.HashSet; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(VertxUnitRunner.class) -public class MatchKeyJavaScriptTest { - - static Vertx vertx; - - static HttpServer httpServer; - - static int PORT = 9230; - - static String HOSTPORT = "http://localhost:" + PORT; - - @BeforeClass - public static void beforeClass(TestContext context) { - vertx = Vertx.vertx(); - Router router = Router.router(vertx); - router.get("/lib/mult.js").handler(ctx -> { - HttpServerResponse response = ctx.response(); - response.setStatusCode(200); - response.putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - response.end("function mult(p1, p2) { return p1 * p2; }"); - }); - router.get("/lib/syntaxerror.js").handler(ctx -> { - HttpServerResponse response = ctx.response(); - response.setStatusCode(200); - response.putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - response.end("function mult(p1, p2) { return p1 *; }"); - }); - router.get("/lib/isbn-match.js").handler(ctx -> { - HttpServerResponse response = ctx.response(); - response.setStatusCode(200); - response.putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - response.end("x => {" - + "var identifiers = JSON.parse(x).identifiers;" - + "const isbn = [];" - + "for (i = 0; i < identifiers.length; i++) {" - + " isbn.push(identifiers[i].isbn);" - + "}" - + "return isbn;" - + "}"); - }); - router.get("/lib/isbn-match.mjs").handler(ctx -> { - HttpServerResponse response = ctx.response(); - response.setStatusCode(200); - response.putHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - response.end("export function matchkey(x) {" - + "var identifiers = JSON.parse(x).identifiers;" - + "const isbn = [];" - + "for (let i = 0; i < identifiers.length; i++) {" - + " isbn.push(identifiers[i].isbn);" - + "}" - + "return isbn;" - + "}"); - }); - httpServer = vertx.createHttpServer(); - httpServer.requestHandler(router).listen(PORT).onComplete(context.asyncAssertSuccess()); - } - - @AfterClass - public static void afterClass(TestContext context) { - vertx.close().onComplete(context.asyncAssertSuccess()); - } - - @Test - public void testMissingConfig(TestContext context) { - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject()) - .onComplete(context.asyncAssertFailure(e -> - assertThat(e.getMessage(), is("javascript: url or script must be given")) - )); - } - - @Test - public void testBadJavaScript(TestContext context) { - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject().put("script", "x =>")) - .onComplete(context.asyncAssertFailure(e -> - assertThat(e.getMessage(), containsString("Expected an operand but found eof")) - )); - } - - @Test - public void testLong(TestContext context) { - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("script", "x => JSON.parse(x).id + 1")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(new JsonObject().put("id", 2), keys); - assertThat(keys, containsInAnyOrder("3")); - })); - } - - @Test - public void testString(TestContext context) { - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("script", "x => JSON.parse(x).id + 'x'")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(new JsonObject().put("id", "2"), keys); - assertThat(keys, containsInAnyOrder("2x")); - })); - } - - @Test - public void testBoolean(TestContext context) { - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("script", "x => JSON.parse(x).id > 1")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(new JsonObject().put("id", "2"), keys); - assertThat(keys, containsInAnyOrder()); - })); - } - - @Test - public void testArray(TestContext context) { - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("script", "function mult(p1, p2) { return p1 * p2; };" - + " x => [JSON.parse(x).id, mult(2, 3)]")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(new JsonObject().put("id", "2"), keys); - assertThat(keys, containsInAnyOrder("6", "2")); - })); - } - - @Test - public void testNotFound(TestContext context) { - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("url", HOSTPORT + "/lib/notfound.js")) - .onComplete(context.asyncAssertFailure(e -> - assertThat(e.getMessage(), is("Response status code 404 is not equal to 200")) - )); - } - - @Test - public void testSyntaxError(TestContext context) { - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("url", HOSTPORT + "/lib/syntaxerror.js")) - .onComplete(context.asyncAssertFailure(e -> { - assertThat(e.getMessage(), containsString("SyntaxError")); - m.close(); - })); - } - - @Test - public void testIsbnMatchUrl(TestContext context) { - JsonObject inventory = new JsonObject() - .put("identifiers", new JsonArray() - .add(new JsonObject() - .put("isbn", "73209629")) - .add(new JsonObject() - .put("isbn", "73209623")) - - ); - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("url", HOSTPORT + "/lib/isbn-match.js")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(inventory, keys); - assertThat(keys, containsInAnyOrder("73209629", "73209623")); - m.close(); - m.close(); - })); - } - - @Test - public void testIsbnMatchUrlModule(TestContext context) { - JsonObject inventory = new JsonObject() - .put("identifiers", new JsonArray() - .add(new JsonObject() - .put("isbn", "73209629")) - .add(new JsonObject() - .put("isbn", "73209623")) - - ); - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("url", HOSTPORT + "/lib/isbn-match.mjs")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(inventory, keys); - assertThat(keys, containsInAnyOrder("73209629", "73209623")); - m.close(); - m.close(); - })); - } - - @Test - public void testUrlAndScript(TestContext context) { - Collection keys = new HashSet<>(); - MatchKeyMethod m = new MatchKeyJavaScript(); - m.configure(vertx, new JsonObject() - .put("url", HOSTPORT + "/lib/mult.js") - .put("script", "x => [JSON.parse(x).id, mult(2, 3)]")) - .onComplete(context.asyncAssertSuccess(x -> { - m.getKeys(new JsonObject().put("id", "2"), keys); - assertThat(keys, containsInAnyOrder("6", "2")); - m.close(); - })); - } -} diff --git a/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPathTest.java b/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPathTest.java deleted file mode 100644 index 4ada642e..00000000 --- a/server/src/test/java/com/indexdata/reservoir/matchkey/impl/MatchKeyJsonPathTest.java +++ /dev/null @@ -1,175 +0,0 @@ -package com.indexdata.reservoir.matchkey.impl; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.is; - -import com.indexdata.reservoir.matchkey.MatchKeyMethod; -import com.jayway.jsonpath.InvalidPathException; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.unit.TestContext; -import io.vertx.ext.unit.junit.VertxUnitRunner; -import java.util.HashSet; -import java.util.Set; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(VertxUnitRunner.class) -public class MatchKeyJsonPathTest { - - static Vertx vertx; - - @BeforeClass - public static void beforeClass() { - vertx = Vertx.vertx(); - } - - @AfterClass - public static void afterClass(TestContext context) { - vertx.close().onComplete(context.asyncAssertSuccess()); - } - - @Test - public void matchKeyJsonPathNonConfigured() { - MatchKeyMethod matchKeyMethod = new MatchKeyJsonPath(); - JsonObject payload = new JsonObject(); - Set keys = new HashSet<>(); - Exception e = Assert.assertThrows( - IllegalArgumentException.class, - () -> matchKeyMethod.getKeys(payload, keys)); - assertThat(e.getMessage(), is("path can not be null")); - } - - @Test - public void matchKeyBadPath(TestContext context) { - MatchKeyMethod matchKeyMethod = new MatchKeyJsonPath(); - matchKeyMethod.configure(vertx, new JsonObject()).onComplete(context.asyncAssertFailure(e -> - assertThat(e.getMessage(), is("jsonpath: expr must be given")) - )); - } - - @Test - public void matchKeyJsonPathConfigureInvalidJsonPath() { - MatchKeyMethod m = new MatchKeyJsonPath(); - JsonObject configuration = new JsonObject().put("expr", "$.fields.010.subfields[x"); - Assert.assertThrows(InvalidPathException.class, - () -> m.configure(vertx, configuration)); - } - - @Test - public void matchKeyJsonPathConfigureMarc(TestContext context) { - MatchKeyMethod m = new MatchKeyJsonPath(); - m.configure(vertx, new JsonObject().put("expr", "$.marc.fields[*].010.subfields[*].a")) - .onComplete(context.asyncAssertSuccess(s -> { - - JsonObject payload = new JsonObject() - .put("marc", new JsonObject() - .put("leader", "00942nam 22002531a 4504") - .put("fields", new JsonArray() - .add(new JsonObject().put("001", " 73209622 //r823")) - .add(new JsonObject().put("010", new JsonObject() - .put("subfields", new JsonArray() - .add(new JsonObject().put("b", "73209622")) - ) - )) - .add(new JsonObject().put("245", new JsonObject() - .put("subfields", new JsonArray() - .add(new JsonObject().put("a", "The Computer Bible /")) - .add(new JsonObject().put("c", "J. Arthur Baird, David Noel Freedman, editors." )) - ) - )) - ) - ); - Set keys = new HashSet<>(); - m.getKeys(payload, keys); - assertThat(keys, is(empty())); - - payload = new JsonObject() - .put("marc", new JsonObject() - .put("leader", "00942nam 22002531a 4504") - .put("fields", new JsonArray() - .add(new JsonObject().put("001", " 73209622 //r823")) - .add(new JsonObject().put("010", new JsonObject() - .put("subfields", new JsonArray() - .add(new JsonObject().put("a", "73209622")) - .add(new JsonObject().put("a", "73209623")) - ) - )) - .add(new JsonObject().put("245", new JsonObject() - .put("subfields", new JsonArray() - .add(new JsonObject().put("a", "The Computer Bible /")) - .add(new JsonObject().put("c", "J. Arthur Baird, David Noel Freedman, editors." )) - ) - )) - )); - keys.clear(); - m.getKeys(payload, keys); - assertThat(keys, containsInAnyOrder("73209622", "73209623")); - })); - } - - @Test - public void matchKeyJsonPathConfigureInventory(TestContext context) { - MatchKeyMethod m = new MatchKeyJsonPath(); - m.configure(vertx, new JsonObject().put("expr", "$.inventory.isbn[*]")) - .onComplete(context.asyncAssertSuccess(s -> { - JsonObject payload = new JsonObject() - .put("inventory", new JsonObject() - .put("isbn", new JsonArray().add("73209622"))); - Set keys = new HashSet<>(); - m.getKeys(payload, keys); - assertThat(keys, contains("73209622")); - - payload = new JsonObject() - .put("inventory", new JsonObject() - .put("issn", new JsonArray().add("73209622"))); - keys.clear(); - m.getKeys(payload, keys); - assertThat(keys, is(empty())); - })); - } - - Future matchKeyVerify(String pattern, Set expectedKeys, JsonObject payload) { - MatchKeyMethod m = new MatchKeyJsonPath(); - return m.configure(vertx, new JsonObject().put("expr", pattern)) - .map(matchKeyMethod -> { - Set keys = new HashSet<>(); - m.getKeys(payload, keys); - Assert.assertEquals(expectedKeys, keys); - return null; - }); - } - - @Test - public void matchKeyJsonPathExpressions(TestContext context) { - JsonObject inventory = new JsonObject() - .put("identifiers", new JsonArray() - .add(new JsonObject() - .put("isbn", "73209629")) - .add(new JsonObject() - .put("isbn", "73209623")) - - ) - .put("matchKey", new JsonObject() - .put("title", "Panisci fistula") - .put("remainder-of-title", " : tre preludi per tre flauti") - .put("medium", "[sound recording]") - ) - ; - - matchKeyVerify("$.identifiers[*].isbn", Set.of("73209629", "73209623"), inventory) - .compose(x -> matchKeyVerify("$.matchKey.title", Set.of("Panisci fistula"), inventory)) - .compose(x -> matchKeyVerify("$.matchKey", Set.of(), inventory)) - .compose(x -> matchKeyVerify("$.matchKey[?(@.title)]", Set.of(), inventory)) - .onComplete(context.asyncAssertSuccess()); - } - -} diff --git a/server/src/test/java/com/indexdata/reservoir/server/MainVerticleTest.java b/server/src/test/java/com/indexdata/reservoir/server/MainVerticleTest.java index 9195f140..f4d4ac96 100644 --- a/server/src/test/java/com/indexdata/reservoir/server/MainVerticleTest.java +++ b/server/src/test/java/com/indexdata/reservoir/server/MainVerticleTest.java @@ -73,18 +73,13 @@ public class MainVerticleTest extends TestBase { @After public void after() { - deleteIssnMatchKey(); RestAssured.given() .header(XOkapiHeaders.TENANT, TENANT_1) .delete("/reservoir/config/oai") .then() .statusCode(204); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .delete("/reservoir/config/matchkeys/issn"); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .delete("/reservoir/config/matchkeys/isbn"); + deleteIssnMatchKey(); + deleteIsbnMatchKey(); RestAssured.given() .header(XOkapiHeaders.TENANT, TENANT_1) .header("Content-Type", "application/json") @@ -103,7 +98,6 @@ public void after() { .delete("/reservoir/pmh-clients/" + PMH_CLIENT_ID); } - public class SruVerify { List identifiers = new LinkedList<>(); @@ -303,213 +297,61 @@ public void ingestTitleUnknownTenant() { } @Test - public void matchKeysNonExistingMethod() { - JsonObject matchKey = new JsonObject() - .put("id", "xx") - .put("method", "other") - .put("params", new JsonObject()); + public void matchKeysNonExistingMatcherModule() { + JsonObject matchKey = new JsonObject() + .put("id", "xx") + .put("matcher", "not-exists"); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .post("/reservoir/config/matchkeys") - .then().statusCode(400) - .contentType("text/plain") - .body(Matchers.is("Non-existing method 'other'")); + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .body(matchKey.encode()) + .post("/reservoir/config/matchkeys") + .then().statusCode(400) + .contentType("text/plain") + .body(Matchers.is("Matcher module 'not-exists' does not exist")); } - @Test - public void matchKeysNonExistingMatcherModule() { - JsonObject matchKey = new JsonObject() - .put("id", "xx") - .put("matcher", "not-exists"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .post("/reservoir/config/matchkeys") - .then().statusCode(400) - .contentType("text/plain") - .body(Matchers.is("Matcher module 'not-exists' does not exist")); - } - - @Test - public void testMatchKeysExistingMatcherModuleWithInvocation() { - JsonObject module = new JsonObject() - .put("id", "exists") - .put("type", "jsonpath") - .put("script", "$.marc"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(module.encode()) - .post("/reservoir/config/modules") - .then() - .statusCode(201) - .contentType("application/json"); - - JsonObject matchKey = new JsonObject() - .put("id", "works") - .put("matcher", "exists::function"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .post("/reservoir/config/matchkeys") - .then() - .statusCode(201) - .contentType("application/json"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .delete("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(204); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .delete("/reservoir/config/modules/" + module.getString("id")) - .then().statusCode(204); - } - @Test - public void testMatchkeysCrudWithMethod() { - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .param("count", "none") - .get("/reservoir/config/matchkeys") - .then().statusCode(200) - .contentType("application/json") - .body("matchKeys", is(empty())) - .body("resultInfo.totalRecords", is(nullValue())); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .param("count", "foo") - .get("/reservoir/config/matchkeys") - .then().statusCode(400) - .contentType("text/plain") - .body(containsString("Instance does not match any of [\"exact\",\"none\"]")); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .param("count", "exact") - .get("/reservoir/config/matchkeys") - .then().statusCode(200) - .contentType("application/json") - .body("matchKeys", is(empty())) - .body("resultInfo.totalRecords", is(0)); + public void testMatchKeysExistingMatcherModuleWithInvocation() { + JsonObject module = new JsonObject() + .put("id", "exists") + .put("type", "jsonpath") + .put("script", "$.marc"); - JsonObject matchKey = new JsonObject() - .put("id", "10a") - .put("method", "jsonpath") - .put("params", new JsonObject().put("marc", "$.fields.010.subfields[*].a")) - .put("update", "ingest"); - - JsonObject matchKeyOut = new JsonObject() - .put("id", "10a") - .put("matcher", null) - .put("method", "jsonpath") - .put("params", new JsonObject().put("marc", "$.fields.010.subfields[*].a")) - .put("update", "ingest"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .get("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(404) - .contentType("text/plain") - .body(Matchers.is("MatchKey " + matchKeyOut.getString("id") + " not found")); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .post("/reservoir/config/matchkeys") - .then().statusCode(201) - .contentType("application/json") - .body(Matchers.is(matchKey.encode())); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .post("/reservoir/config/matchkeys") - .then().statusCode(400) - .contentType("text/plain") - .body(containsString("duplicate key value violates unique constraint")); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .get("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(200) - .contentType("application/json") - .body(Matchers.is(matchKeyOut.encode())); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .get("/reservoir/config/matchkeys") - .then().statusCode(200) - .contentType("application/json") - .body("matchKeys", hasSize(1)) - .body("matchKeys[0].id", is(matchKeyOut.getString("id"))) - .body("matchKeys[0].method", is(matchKeyOut.getString("method"))) - .body("matchKeys[0].update", is(matchKeyOut.getString("update"))) - .body("matchKeys[0].matcher", is(matchKeyOut.getString("matcher"))); - // should really check that params are same - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .get("/reservoir/config/matchkeys?query=method=" + matchKeyOut.getString("method")) - .then().statusCode(200) - .contentType("application/json") - .body("matchKeys", hasSize(1)) - .body("matchKeys[0].id", is(matchKeyOut.getString("id"))) - .body("matchKeys[0].method", is(matchKeyOut.getString("method"))) - .body("matchKeys[0].update", is(matchKeyOut.getString("update"))); - - matchKey.put("update", "manual"); - matchKeyOut.put("update", "manual"); - - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .put("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(204); + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .body(module.encode()) + .post("/reservoir/config/modules") + .then() + .statusCode(201) + .contentType("application/json"); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .get("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(200) - .contentType("application/json") - .body(Matchers.is(matchKeyOut.encode())); + JsonObject matchKey = new JsonObject() + .put("id", "works") + .put("matcher", "exists::function"); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .delete("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(204); + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .body(matchKey.encode()) + .post("/reservoir/config/matchkeys") + .then() + .statusCode(201) + .contentType("application/json"); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .get("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(404); + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .delete("/reservoir/config/matchkeys/" + matchKey.getString("id")) + .then().statusCode(204); - RestAssured.given() - .header(XOkapiHeaders.TENANT, TENANT_1) - .header("Content-Type", "application/json") - .body(matchKey.encode()) - .put("/reservoir/config/matchkeys/" + matchKey.getString("id")) - .then().statusCode(404); + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .delete("/reservoir/config/modules/" + module.getString("id")) + .then().statusCode(204); } @Test @@ -565,13 +407,6 @@ public void testMatchkeyCrudWithMatcher() { .put("type", "jsonpath") .put("script", "$.marc.fields.010.subfields[*].a"); - JsonObject module10aOut = new JsonObject() - .put("id", "matcher-10a") - .put("type", "jsonpath") - .put("url", null) - .put("function", null) - .put("script", "$.marc.fields.010.subfields[*].a"); - //post module first RestAssured.given() .header(XOkapiHeaders.TENANT, TENANT_1) @@ -581,7 +416,7 @@ public void testMatchkeyCrudWithMatcher() { .then() .statusCode(201) .contentType("application/json") - .body(Matchers.is(module10aOut.encode())); + .body(Matchers.is(module10a.encode())); RestAssured.given() .header(XOkapiHeaders.TENANT, TENANT_1) @@ -817,8 +652,6 @@ public void ingestRecords() { .param("query", "cql.allRecords=true") .delete("/reservoir/records") .then().statusCode(204); - - } static String verifyOaiResponseRuntime(String s, String verb, List identifiers, int length, JsonArray expRecords) { @@ -1332,12 +1165,6 @@ JsonObject createIssnMatchKey(String update) { .put("type", "jsonpath") .put("script", "$.inventory.issn[*]"); - JsonObject issnMatcherOut = new JsonObject() - .put("id", "issn-matcher") - .put("type", "jsonpath") - .put("url", null) - .put("function", null) - .put("script", "$.inventory.issn[*]"); //post module first RestAssured.given() @@ -1346,9 +1173,9 @@ JsonObject createIssnMatchKey(String update) { .body(issnMatcher.encode()) .post("/reservoir/config/modules") .then() - //.statusCode(201) - //.contentType("application/json") - .body(Matchers.is(issnMatcherOut.encode())); + .statusCode(201) + .contentType("application/json") + .body(Matchers.is(issnMatcher.encode())); JsonObject matchKey = new JsonObject() .put("id", "issn") @@ -1379,15 +1206,41 @@ void deleteIssnMatchKey() { .then().statusCode(anyOf(is(204), is(404))); } + void deleteIsbnMatchKey() { + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .delete("/reservoir/config/modules/isbn-matcher") + .then().statusCode(anyOf(is(204), is(404))); + + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .delete("/reservoir/config/matchkeys/isbn") + .then().statusCode(anyOf(is(204), is(404))); + } + + JsonObject createIsbnMatchKey() { return createIsbnMatchKey(null); } JsonObject createIsbnMatchKey(String updateValue) { + JsonObject matchModule = new JsonObject() + .put("id", "isbn-matcher") + .put("type", "jsonpath") + .put("script", "$.inventory.isbn[*]"); + + RestAssured.given() + .header(XOkapiHeaders.TENANT, TENANT_1) + .header("Content-Type", "application/json") + .body(matchModule.encode()) + .post("/reservoir/config/modules") + .then().statusCode(201) + .contentType("application/json") + .body(Matchers.is(matchModule.encode())); + JsonObject matchKey = new JsonObject() .put("id", "isbn") - .put("method", "jsonpath") - .put("params", new JsonObject().put("expr", "$.inventory.isbn[*]")); + .put("matcher", "isbn-matcher"); if (updateValue != null) { matchKey.put("update", updateValue);