Skip to content

Commit 6e9f453

Browse files
aliszkaaliszka
authored
Merge pull request #65 from semi-technologies/WVT-95_batch_delete_by_filter
WVT-95: batch delete by filter
2 parents 5240faa + 5b0c332 commit 6e9f453

File tree

8 files changed

+340
-22
lines changed

8 files changed

+340
-22
lines changed

‎batch/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import ObjectsBatcher from "./objectsBatcher";
2+
import ObjectsBatchDeleter from "./objectsBatchDeleter";
23
import ReferencesBatcher from "./referencesBatcher";
34
import ReferencePayloadBuilder from "./referencePayloadBuilder";
45

56
const batch = (client) => {
67
return {
78
objectsBatcher: () => new ObjectsBatcher(client),
9+
objectsBatchDeleter: () => new ObjectsBatchDeleter(client),
810
referencesBatcher: () => new ReferencesBatcher(client),
911
referencePayloadBuilder: () => new ReferencePayloadBuilder(client),
1012
};

‎batch/journey.test.js‎

Lines changed: 214 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
const { Operator } = require("../filters/operator");
12
const weaviate = require("../index");
3+
const { Output, Status } = require("./objectsBatchDeleter");
24

35
const thingClassName = "BatchJourneyTestThing";
46
const otherThingClassName = "BatchJourneyTestOtherThing";
@@ -241,6 +243,171 @@ describe("batch importing", () => {
241243
it("tears down and cleans up", () => cleanup(client));
242244
});
243245

246+
describe("batch deleting", () => {
247+
const client = weaviate.client({
248+
scheme: "http",
249+
host: "localhost:8080",
250+
});
251+
252+
it("sets up schema", () => setup(client));
253+
it("sets up data", () => setupData(client));
254+
255+
it("batch deletes with dryRun and verbose output", () =>
256+
client.batch
257+
.objectsBatchDeleter()
258+
.withClassName(thingClassName)
259+
.withWhere({
260+
operator: Operator.EQUAL,
261+
valueString: "bar1",
262+
path: ["stringProp"]
263+
})
264+
.withDryRun(true)
265+
.withOutput(Output.VERBOSE)
266+
.do()
267+
.then(result => {
268+
expect(result.dryRun).toBe(true);
269+
expect(result.output).toBe(Output.VERBOSE);
270+
expect(result.match).toEqual({
271+
class: thingClassName,
272+
where: {
273+
operands: null, // FIXME should not be received
274+
operator: Operator.EQUAL,
275+
valueString: "bar1",
276+
path: ["stringProp"],
277+
},
278+
})
279+
expect(result.results).toEqual({
280+
successful: 0,
281+
failed: 0,
282+
matches: 1,
283+
limit: 10000,
284+
objects: [{
285+
id: thingIds[1],
286+
status: Status.DRYRUN,
287+
}],
288+
});
289+
})
290+
)
291+
292+
it("batch deletes with dryRun and minimal output", () =>
293+
client.batch
294+
.objectsBatchDeleter()
295+
.withClassName(otherThingClassName)
296+
.withWhere({
297+
operator: Operator.LIKE,
298+
valueString: "foo3",
299+
path: ["stringProp"]
300+
})
301+
.withDryRun(true)
302+
.withOutput(Output.MINIMAL)
303+
.do()
304+
.then(result => {
305+
expect(result.dryRun).toBe(true);
306+
expect(result.output).toBe(Output.MINIMAL);
307+
expect(result.match).toEqual({
308+
class: otherThingClassName,
309+
where: {
310+
operands: null, // FIXME should not be received
311+
operator: Operator.LIKE,
312+
valueString: "foo3",
313+
path: ["stringProp"],
314+
},
315+
})
316+
expect(result.results).toEqual({
317+
successful: 0,
318+
failed: 0,
319+
matches: 1,
320+
limit: 10000,
321+
objects: null,
322+
});
323+
})
324+
)
325+
326+
it("batch deletes but no matches with default dryRun and output", () =>
327+
client.batch
328+
.objectsBatchDeleter()
329+
.withClassName(otherThingClassName)
330+
.withWhere({
331+
operator: Operator.EQUAL,
332+
valueString: "doesNotExist",
333+
path: ["stringProp"]
334+
})
335+
.do()
336+
.then(result => {
337+
expect(result.dryRun).toBe(false);
338+
expect(result.output).toBe(Output.MINIMAL);
339+
expect(result.match).toEqual({
340+
class: otherThingClassName,
341+
where: {
342+
operands: null, // FIXME should not be received
343+
operator: Operator.EQUAL,
344+
valueString: "doesNotExist",
345+
path: ["stringProp"],
346+
},
347+
})
348+
expect(result.results).toEqual({
349+
successful: 0,
350+
failed: 0,
351+
matches: 0,
352+
limit: 10000,
353+
objects: null,
354+
});
355+
})
356+
)
357+
358+
it("batch deletes with default dryRun", () => {
359+
const inAMinute = "" + (new Date().getTime() + 60 * 1000);
360+
return client.batch
361+
.objectsBatchDeleter()
362+
.withClassName(otherThingClassName)
363+
.withWhere({
364+
operator: Operator.LESS_THAN,
365+
valueString: inAMinute,
366+
path: ["_creationTimeUnix"]
367+
})
368+
.withOutput(Output.VERBOSE)
369+
.do()
370+
.then(result => {
371+
expect(result.dryRun).toBe(false);
372+
expect(result.output).toBe(Output.VERBOSE);
373+
expect(result.match).toEqual({
374+
class: otherThingClassName,
375+
where: {
376+
operands: null, // FIXME should not be received
377+
operator: Operator.LESS_THAN,
378+
valueString: inAMinute,
379+
path: ["_creationTimeUnix"],
380+
},
381+
})
382+
expect(result.results.successful).toBe(2);
383+
expect(result.results.failed).toBe(0);
384+
expect(result.results.matches).toBe(2);
385+
expect(result.results.limit).toBe(10000);
386+
expect(result.results.objects).toHaveLength(2)
387+
expect(result.results.objects).toContainEqual({
388+
id: otherThingIds[0],
389+
status: Status.SUCCESS,
390+
});
391+
expect(result.results.objects).toContainEqual({
392+
id: otherThingIds[1],
393+
status: Status.SUCCESS,
394+
});
395+
})
396+
})
397+
398+
it("batch deletes fails due to validation", () =>
399+
client.batch
400+
.objectsBatchDeleter()
401+
.withClassName("")
402+
.withWhere("shouldBeObject")
403+
.do()
404+
.catch(err => expect(err.toString()).toBe("Error: invalid usage: string className must be set - set with .withClassName(className), object where must be set - set with .withWhere(whereFilter)"))
405+
)
406+
407+
it("tears down and cleans up", () => cleanup(client));
408+
});
409+
410+
244411
const setup = async (client) => {
245412
// first import the classes
246413
await Promise.all([
@@ -266,6 +433,9 @@ const setup = async (client) => {
266433
dataType: ["string"],
267434
},
268435
],
436+
invertedIndexConfig: {
437+
indexTimestamps: true,
438+
},
269439
})
270440
.do(),
271441
]);
@@ -280,7 +450,50 @@ const setup = async (client) => {
280450
.do();
281451
};
282452

283-
//
453+
const setupData = (client) => {
454+
const toImport = [
455+
{
456+
class: thingClassName,
457+
id: thingIds[0],
458+
properties: { stringProp: "foo1" },
459+
},
460+
{
461+
class: thingClassName,
462+
id: thingIds[1],
463+
properties: { stringProp: "bar1" },
464+
},
465+
{
466+
class: thingClassName,
467+
id: thingIds[2],
468+
properties: { stringProp: "foo2" },
469+
},
470+
{
471+
class: thingClassName,
472+
id: thingIds[3],
473+
properties: { stringProp: "bar2" },
474+
},
475+
{
476+
class: otherThingClassName,
477+
id: otherThingIds[0],
478+
properties: { stringProp: "foo3" },
479+
},
480+
{
481+
class: otherThingClassName,
482+
id: otherThingIds[1],
483+
properties: { stringProp: "bar3" },
484+
},
485+
];
486+
487+
return client.batch
488+
.objectsBatcher()
489+
.withObject(toImport[0])
490+
.withObject(toImport[1])
491+
.withObject(toImport[2])
492+
.withObject(toImport[3])
493+
.withObject(toImport[4])
494+
.withObject(toImport[5])
495+
.do();
496+
}
284497

285498
const cleanup = (client) =>
286499
Promise.all([

‎batch/objectsBatchDeleter.js‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { isValidStringProperty } from "../validation/string";
2+
3+
export const Output = {
4+
VERBOSE: "verbose",
5+
MINIMAL: "minimal",
6+
};
7+
8+
export const Status = {
9+
SUCCESS: "SUCCESS",
10+
FAILED: "FAILED",
11+
DRYRUN: "DRYRUN",
12+
};
13+
14+
export default class ObjectsBatchDeleter {
15+
className;
16+
whereFilter;
17+
output;
18+
dryRun;
19+
20+
constructor(client) {
21+
this.client = client;
22+
this.errors = [];
23+
}
24+
25+
withClassName(className) {
26+
this.className = className;
27+
return this;
28+
}
29+
30+
withWhere(whereFilter) {
31+
this.whereFilter = whereFilter;
32+
return this;
33+
}
34+
35+
withOutput(output) {
36+
this.output = output;
37+
return this;
38+
}
39+
40+
withDryRun(dryRun) {
41+
this.dryRun = dryRun;
42+
return this;
43+
}
44+
45+
payload() {
46+
return {
47+
match: {
48+
class: this.className,
49+
where: this.whereFilter,
50+
},
51+
output: this.output,
52+
dryRun: this.dryRun,
53+
}
54+
}
55+
56+
validateClassName() {
57+
if (!isValidStringProperty(this.className)) {
58+
this.errors = [
59+
...this.errors,
60+
"string className must be set - set with .withClassName(className)",
61+
];
62+
}
63+
}
64+
65+
validateWhereFilter() {
66+
if (typeof this.whereFilter != "object") {
67+
this.errors = [
68+
...this.errors,
69+
"object where must be set - set with .withWhere(whereFilter)"
70+
]
71+
}
72+
}
73+
74+
validate() {
75+
this.validateClassName();
76+
this.validateWhereFilter();
77+
};
78+
79+
do() {
80+
this.validate();
81+
if (this.errors.length > 0) {
82+
return Promise.reject(
83+
new Error("invalid usage: " + this.errors.join(", "))
84+
);
85+
}
86+
const path = `/batch/objects`;
87+
return this.client.delete(path, this.payload(), true);
88+
};
89+
}

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3.4'
22
services:
33
weaviate:
44
# TODO: temporarily use this version until next release
5-
image: semitechnologies/weaviate:1.12.2-779bfd9
5+
image: semitechnologies/weaviate:latest@sha256:5552cbb7f319d8c9e7afeef87e1152906fab04e4f99e2c07a9ad46293d6a6870
66
restart: on-failure:0
77
ports:
88
- "8080:8080"

‎filters/operator.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
export const Operator = {
2+
AND: "And",
3+
LIKE: "Like",
4+
OR: "Or",
5+
EQUAL: "Equal",
6+
NOT: "Not",
7+
NOT_EQUAL: "NotEqual",
8+
GREATER_THAN: "GreaterThan",
9+
GREATER_THAN_EQUAL: "GreaterThanEqual",
10+
LESS_THAN: "LessThan",
11+
LESS_THAN_EQUAL: "LessThanEqual",
12+
WITHIN_GEO_RANGE: "WithinGeoRange",
13+
}

0 commit comments

Comments
 (0)