Lucene: index filters support

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/StandardIndexMaintainer.java

@@ -359,49 +359,18 @@ public <M extends Message> List<IndexEntry> filteredIndexEntries(@Nullable final
         if (savedRecord == null) {
             return null;
         }
-        // Apply both filters:
-        // 1. Index predicates (if exist)
-        // 2. IndexMaintenanceFilter
-        // In the longer term, we will probably think about deprecating the index maintenance filter.
-        final FDBStoreTimer timer = state.store.getTimer();
-        final IndexPredicate predicate = state.index.getPredicate();
-        if (predicate != null) {
-            final long startTime = System.nanoTime();
-            final boolean useMe = predicate.shouldIndexThisRecord(state.store, savedRecord);
-            // Note: for now, IndexPredicate will not support filtering of certain index entries
-            if (timer != null) {
-                final FDBStoreTimer.Events event =
-                        useMe ?
-                        FDBStoreTimer.Events.USE_INDEX_RECORD_BY_PREDICATE :
-                        FDBStoreTimer.Events.SKIP_INDEX_RECORD_BY_PREDICATE;
-                timer.recordSinceNanoTime(event, startTime);
-            }
-            if (!useMe) {
-                // Here: index predicate filters out this record
-                return null;
-            }
-        }
-        final Message record = savedRecord.getRecord();
-        long startTime = System.nanoTime();
-        boolean filterIndexKeys = false;
-        switch (state.filter.maintainIndex(state.index, record)) {
-            case NONE:
-                if (timer != null) {
-                    timer.recordSinceNanoTime(FDBStoreTimer.Events.SKIP_INDEX_RECORD, startTime);
-                }
-                return null;
-            case SOME:
-                filterIndexKeys = true;
-                break;
-            case ALL:
-            default:
-                break;
+        final IndexMaintenanceFilter.IndexValues filterType = getFilterTypeForRecord(savedRecord);
+        if (filterType == IndexMaintenanceFilter.IndexValues.NONE) {
+            return null;
         }
         List<IndexEntry> indexEntries = evaluateIndex(savedRecord);
-        if (!filterIndexKeys) {
+        if (filterType == IndexMaintenanceFilter.IndexValues.ALL) {
             return indexEntries;
         }
+        // Here: filterType is SOME. Check each index entry
+        long startTime = System.nanoTime();
         int i = 0;
+        final Message record = savedRecord.getRecord();
         while (i < indexEntries.size()) {
             if (state.filter.maintainIndexValue(state.index, record, indexEntries.get(i))) {
                 i++;
@@ -418,6 +387,37 @@ public <M extends Message> List<IndexEntry> filteredIndexEntries(@Nullable final
         return indexEntries;
     }
 
+    protected <M extends Message> IndexMaintenanceFilter.IndexValues getFilterTypeForRecord(@Nonnull final FDBIndexableRecord<M> savedRecord) {
+        // Apply both filters:
+        // 1. Index predicates (if exist) - currently supports filtering out (i.e. NONE). If not filtered out, fallthrough to the next filter
+        // 2. IndexMaintenanceFilter - supports ALL, NONE, and SOME
+        // In the longer term, we will probably think about deprecating the index maintenance filter.
+        final FDBStoreTimer timer = state.store.getTimer();
+        final IndexPredicate predicate = state.index.getPredicate();
+        if (predicate != null) {
+            final long startTime = timer != null ? System.nanoTime() : 0L;
+            final boolean useMe = predicate.shouldIndexThisRecord(state.store, savedRecord);
+            // Note: for now, IndexPredicate will not support filtering of certain index entries
+            if (timer != null) {
+                final FDBStoreTimer.Events event =
+                        useMe ?
+                        FDBStoreTimer.Events.USE_INDEX_RECORD_BY_PREDICATE :
+                        FDBStoreTimer.Events.SKIP_INDEX_RECORD_BY_PREDICATE;
+                timer.recordSinceNanoTime(event, startTime);
+            }
+            if (!useMe) {
+                return IndexMaintenanceFilter.IndexValues.NONE;
+            }
+        }
+        long startTime = System.nanoTime();
+        IndexMaintenanceFilter.IndexValues ret = state.filter.maintainIndex(state.index, savedRecord.getRecord());
+        if (ret == IndexMaintenanceFilter.IndexValues.NONE && timer != null) {
+            // events are backward compatible
+            timer.recordSinceNanoTime(FDBStoreTimer.Events.SKIP_INDEX_RECORD, startTime);
+        }
+        return ret;
+    }
+
     @Nonnull
     protected List<IndexEntry> commonKeys(@Nonnull List<IndexEntry> oldIndexEntries,
                                           @Nonnull List<IndexEntry> newIndexEntries) {

fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java

@@ -56,6 +56,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.IndexDeferredMaintenanceControl;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState;
+import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter;
 import com.apple.foundationdb.record.provider.foundationdb.IndexOperation;
 import com.apple.foundationdb.record.provider.foundationdb.IndexOperationResult;
 import com.apple.foundationdb.record.provider.foundationdb.IndexScanBounds;
@@ -458,9 +459,12 @@ public <M extends Message> CompletableFuture<Void> update(@Nullable FDBIndexable
     }
 
     @Nonnull
-    <M extends Message> CompletableFuture<Void> update(@Nullable FDBIndexableRecord<M> oldRecord,
-                                                       @Nullable FDBIndexableRecord<M> newRecord,
+    <M extends Message> CompletableFuture<Void> update(@Nullable FDBIndexableRecord<M> oldRecordUnfiltered,
+                                                       @Nullable FDBIndexableRecord<M> newRecordUnfiltered,
                                                        @Nullable Integer destinationPartitionIdHint) {
+        FDBIndexableRecord<M> oldRecord = maybeFilterRecord(oldRecordUnfiltered);
+        FDBIndexableRecord<M> newRecord = maybeFilterRecord(newRecordUnfiltered);
+
         LOG.trace("update oldRecord={}, newRecord={}", oldRecord, newRecord);
 
         // Extract information for grouping from old and new records
@@ -507,6 +511,19 @@ private <M extends Message> CompletableFuture<Void> updateRecord(
                         .thenAccept(partitionId -> writeDocument(newRecord, entry, partitionId)));
     }
 
+    @Nullable
+    public <M extends Message> FDBIndexableRecord<M> maybeFilterRecord(FDBIndexableRecord<M> rec) {
+        if (rec != null) {
+            final IndexMaintenanceFilter.IndexValues filterType = getFilterTypeForRecord(rec);
+            if (filterType == IndexMaintenanceFilter.IndexValues.NONE) {
+                return null;
+            } else if (filterType == IndexMaintenanceFilter.IndexValues.SOME) {
+                throw new RecordCoreException("Lucene does not support this kind of filtering");
+            }
+        }
+        return rec;
+    }
+
     /**
      * convenience wrapper that calls {@link #tryDelete(FDBIndexableRecord, Tuple)} only if the index is in
      * {@code WriteOnly} mode.
@@ -782,7 +799,7 @@ public IndexScrubbingTools<?> getIndexScrubbingTools(final IndexScrubbingTools.S
                 final Map<String, String> options = state.index.getOptions();
                 if (Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_ENABLED)) ||
                         Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_V2_ENABLED))) {
-                    return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager);
+                    return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager, this);
                 }
                 return null;
             default:

fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java

@@ -67,10 +67,13 @@ public class LuceneIndexScrubbingToolsMissing extends ValueIndexScrubbingToolsMi
     private final LucenePartitioner partitioner;
     @Nonnull
     private final FDBDirectoryManager directoryManager;
+    @Nonnull
+    private final LuceneIndexMaintainer indexMaintainer;
 
-    public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager) {
+    public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager, @Nonnull LuceneIndexMaintainer indexMaintainer) {
         this.partitioner = partitioner;
         this.directoryManager = directoryManager;
+        this.indexMaintainer = indexMaintainer;
     }
 
 
@@ -100,7 +103,7 @@ public CompletableFuture<Issue> handleOneItem(final FDBRecordStore store, final
         }
 
         final FDBStoredRecord<Message> rec = result.get();
-        if (rec == null || !recordTypes.contains(rec.getRecordType())) {
+        if (!shouldHandleItem(rec)) {
             return CompletableFuture.completedFuture(null);
         }
 
@@ -121,6 +124,13 @@ public CompletableFuture<Issue> handleOneItem(final FDBRecordStore store, final
                 });
     }
 
+    private boolean shouldHandleItem(FDBStoredRecord<Message> rec) {
+        if (rec == null || !recordTypes.contains(rec.getRecordType())) {
+            return false;
+        }
+        return indexMaintainer.maybeFilterRecord(rec) != null;
+    }
+
     @SuppressWarnings("PMD.CloseResource")
     private CompletableFuture<Pair<MissingIndexReason, Tuple>> detectMissingIndexKeys(final FDBRecordStore store, FDBStoredRecord<Message> rec) {
         // Generate synthetic record (if applicable) and return the first detected missing (if any).

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java

@@ -53,7 +53,6 @@
 class LuceneIndexScrubbingTest extends FDBLuceneTestBase {
 
     private TestingIndexMaintainerRegistry registry;
-    private boolean flipBoolean = false;
 
     @BeforeEach
     public void beforeEach() {

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java

@@ -20,10 +20,13 @@
 
 package com.apple.foundationdb.record.lucene;
 
+import com.apple.foundationdb.record.IndexEntry;
 import com.apple.foundationdb.record.RecordMetaData;
 import com.apple.foundationdb.record.RecordMetaDataBuilder;
+import com.apple.foundationdb.record.ScanProperties;
 import com.apple.foundationdb.record.TestRecordsGroupedParentChildProto;
 import com.apple.foundationdb.record.metadata.Index;
+import com.apple.foundationdb.record.metadata.IndexPredicate;
 import com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
@@ -32,6 +35,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore;
 import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer;
 import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord;
+import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexScrubber;
 import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer;
 import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
 import com.apple.foundationdb.record.test.TestKeySpace;
@@ -316,9 +320,10 @@ private LuceneIndexTestValidator getValidator(final Supplier<FDBRecordContext> o
 
     @Nonnull
     static Index addIndex(final boolean isSynthetic, final KeyExpression rootExpression,
-                          final Map<String, String> options, final RecordMetaDataBuilder metaDataBuilder) {
+                          final Map<String, String> options, final RecordMetaDataBuilder metaDataBuilder,
+                          @Nullable IndexPredicate predicate) {
         Index index;
-        index = new Index("joinNestedConcat", rootExpression, LuceneIndexTypes.LUCENE, options);
+        index = new Index("joinNestedConcat", rootExpression, LuceneIndexTypes.LUCENE, options, predicate);
 
         if (isSynthetic) {
             final JoinedRecordTypeBuilder joinBuilder = metaDataBuilder.addJoinedRecordType("JoinChildren");
@@ -399,6 +404,28 @@ public void explicitMergeIndex(final FDBRecordContext context, @Nullable FDBStor
         }
     }
 
+    public long findMissingIndexEntries(final FDBRecordContext context, @Nullable FDBStoreTimer timer) {
+        FDBRecordStore recordStore = Objects.requireNonNull(schemaSetup.apply(context));
+        try (OnlineIndexScrubber indexBuilder = OnlineIndexScrubber.newBuilder()
+                .setRecordStore(recordStore)
+                .setIndex(index)
+                .setTimer(timer)
+                .build()) {
+            return indexBuilder.scrubMissingIndexEntries();
+        }
+    }
+
+    public List<IndexEntry> findAllRecordsByQuery(final FDBRecordContext context, int group) {
+        LuceneQueryClause search = LuceneQuerySearchClause.MATCH_ALL_DOCS_QUERY;
+
+        FDBRecordStore store = Objects.requireNonNull(schemaSetup.apply(context));
+        LuceneScanBounds scanBounds = isGrouped
+                                      ? LuceneIndexTestValidator.groupedSortedTextSearch(store, index, search, null, group)
+                                      : LuceneIndexTestUtils.fullTextSearch(store, index, search, false);
+        return store.scanIndex(index, scanBounds, null, ScanProperties.FORWARD_SCAN)
+                .asList().join();
+    }
+
     public Random getRandom() {
         return random;
     }
@@ -429,6 +456,8 @@ static class Builder {
         private Index index;
         @Nullable
         private RecordMetaData metadata;
+        @Nullable
+        IndexPredicate predicate = null;
 
         public Builder(final long seed, StoreBuilderSupplier storeBuilderSupplier,
                        TestKeySpacePathManagerExtension pathManager) {
@@ -461,6 +490,12 @@ public Builder setPartitionHighWatermark(final int partitionHighWatermark) {
             return this;
         }
 
+        public Builder setPredicate(@Nullable final IndexPredicate predicate) {
+            this.predicate = predicate;
+            metadata = null;
+            return this;
+        }
+
         public Builder setTextGeneratorWithNewRandom(final RandomTextGenerator textGenerator) {
             this.textGenerator = textGenerator.withNewRandom(random);
             return this;
@@ -484,7 +519,7 @@ public LuceneIndexTestDataModel build() {
                 final Map<String, String> options = getOptions();
                 final RecordMetaDataBuilder metaDataBuilder = LuceneIndexTestDataModel.createBaseMetaDataBuilder();
                 final KeyExpression rootExpression = LuceneIndexTestDataModel.createRootExpression(isGrouped, isSynthetic);
-                this.index = LuceneIndexTestDataModel.addIndex(isSynthetic, rootExpression, options, metaDataBuilder);
+                this.index = LuceneIndexTestDataModel.addIndex(isSynthetic, rootExpression, options, metaDataBuilder, predicate);
                 this.metadata = metaDataBuilder.build();
             }
             final Function<FDBRecordContext, FDBRecordStore> schemaSetup = context -> {

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java

@@ -46,6 +46,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState;
+import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter;
 import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer;
 import com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils;
 import com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage;
@@ -55,6 +56,7 @@
 import com.apple.foundationdb.record.util.pair.Pair;
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.tuple.Tuple;
+import com.apple.test.BooleanSource;
 import com.apple.test.RandomSeedSource;
 import com.apple.test.RandomizedTestUtils;
 import com.google.auto.service.AutoService;
@@ -106,6 +108,7 @@
 import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.MAP_DOC;
 import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertSame;
@@ -633,7 +636,6 @@ void luceneOnlineIndexingTestMulti() throws IOException {
         }
     }
 
-
     protected void openRecordStore(FDBRecordContext context, FDBRecordStoreTestBase.RecordMetaDataHook hook) {
         RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsTextProto.getDescriptor());
         metaDataBuilder.getRecordType(COMPLEX_DOC).setPrimaryKey(concatenateFields("group", "doc_id"));
@@ -644,6 +646,23 @@ protected void openRecordStore(FDBRecordContext context, FDBRecordStoreTestBase.
         setupPlanner(null);
     }
 
+    protected void openRecordStoreWithFilter(FDBRecordContext context, FDBRecordStoreTestBase.RecordMetaDataHook hook, boolean filterOut) {
+        RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsTextProto.getDescriptor());
+        metaDataBuilder.getRecordType(COMPLEX_DOC).setPrimaryKey(concatenateFields("group", "doc_id"));
+        hook.apply(metaDataBuilder);
+        final FDBRecordStore.Builder builder = getStoreBuilder(context, metaDataBuilder.getRecordMetaData())
+                .setSerializer(TextIndexTestUtils.COMPRESSING_SERIALIZER);
+        if (filterOut) {
+            recordStore = builder
+                    .setIndexMaintenanceFilter((i, r) -> IndexMaintenanceFilter.IndexValues.NONE)
+                    .createOrOpen();
+        } else {
+            recordStore = builder
+                    .createOrOpen();
+        }
+        setupPlanner(null);
+    }
+
     @ParameterizedTest
     @ValueSource(ints = {1, 2, 3})
     void luceneOnlineIndexingTestGroupingKeys(int groupingCount) {
@@ -760,6 +779,43 @@ void luceneOnlineIndexingTestGroupingKeysBackgroundMerge(int groupingCount) thro
         assertTrue(newLength < oldLength);
     }
 
+    @ParameterizedTest
+    @BooleanSource
+    void luceneOnlineIndexingTestNoMergeIfFilteredOutRecords(boolean filterOut) throws IOException {
+        Index index = new Index(
+                "Map_with_auto_complete$entry-value",
+                new GroupingKeyExpression(field("entry",
+                        KeyExpression.FanType.FanOut).nest(concat(LuceneIndexTestUtils.keys)), 3),
+                LuceneIndexTypes.LUCENE,
+                ImmutableMap.of());
+
+        RecordMetaDataHook hook = metaDataBuilder -> {
+            metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME);
+            TextIndexTestUtils.addRecordTypePrefix(metaDataBuilder);
+            metaDataBuilder.addIndex(MAP_DOC, index);
+        };
+        int group = 3;
+
+        // write/overwrite records
+        boolean needMerge = false;
+        for (int iLast = 60; iLast > 40; iLast --) {
+            try (FDBRecordContext context = openContext()) {
+                openRecordStoreWithFilter(context, hook, filterOut);
+                for (int i = 0; i < iLast; i++) {
+                    recordStore.saveRecord(multiEntryMapDoc(77L * i, ENGINEER_JOKE + iLast, group));
+                }
+                final Set<Index> indexSet = recordStore.getIndexDeferredMaintenanceControl().getMergeRequiredIndexes();
+                if (indexSet != null && !indexSet.isEmpty()) {
+                    assertEquals(1, indexSet.size());
+                    assertEquals(indexSet.stream().findFirst().get().getName(), index.getName());
+                    needMerge = true;
+                }
+                commit(context);
+            }
+        }
+        assertNotEquals(needMerge, filterOut);
+    }
+
     private TestRecordsTextProto.MapDocument multiEntryMapDoc(long id, String text, int group) {
         assertTrue(group < 4);
         String text2 = "Text 2, and " + (id % 2);