diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/StandardIndexMaintainer.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/StandardIndexMaintainer.java index fa6f76b0f1..1d57da9c3d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/StandardIndexMaintainer.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/indexes/StandardIndexMaintainer.java @@ -359,49 +359,18 @@ public List filteredIndexEntries(@Nullable final if (savedRecord == null) { return null; } - // Apply both filters: - // 1. Index predicates (if exist) - // 2. IndexMaintenanceFilter - // In the longer term, we will probably think about deprecating the index maintenance filter. - final FDBStoreTimer timer = state.store.getTimer(); - final IndexPredicate predicate = state.index.getPredicate(); - if (predicate != null) { - final long startTime = System.nanoTime(); - final boolean useMe = predicate.shouldIndexThisRecord(state.store, savedRecord); - // Note: for now, IndexPredicate will not support filtering of certain index entries - if (timer != null) { - final FDBStoreTimer.Events event = - useMe ? - FDBStoreTimer.Events.USE_INDEX_RECORD_BY_PREDICATE : - FDBStoreTimer.Events.SKIP_INDEX_RECORD_BY_PREDICATE; - timer.recordSinceNanoTime(event, startTime); - } - if (!useMe) { - // Here: index predicate filters out this record - return null; - } - } - final Message record = savedRecord.getRecord(); - long startTime = System.nanoTime(); - boolean filterIndexKeys = false; - switch (state.filter.maintainIndex(state.index, record)) { - case NONE: - if (timer != null) { - timer.recordSinceNanoTime(FDBStoreTimer.Events.SKIP_INDEX_RECORD, startTime); - } - return null; - case SOME: - filterIndexKeys = true; - break; - case ALL: - default: - break; + final IndexMaintenanceFilter.IndexValues filterType = getFilterTypeForRecord(savedRecord); + if (filterType == IndexMaintenanceFilter.IndexValues.NONE) { + return null; } List indexEntries = evaluateIndex(savedRecord); - if (!filterIndexKeys) { + if (filterType == IndexMaintenanceFilter.IndexValues.ALL) { return indexEntries; } + // Here: filterType is SOME. Check each index entry + long startTime = System.nanoTime(); int i = 0; + final Message record = savedRecord.getRecord(); while (i < indexEntries.size()) { if (state.filter.maintainIndexValue(state.index, record, indexEntries.get(i))) { i++; @@ -418,6 +387,37 @@ public List filteredIndexEntries(@Nullable final return indexEntries; } + protected IndexMaintenanceFilter.IndexValues getFilterTypeForRecord(@Nonnull final FDBIndexableRecord savedRecord) { + // Apply both filters: + // 1. Index predicates (if exist) - currently supports filtering out (i.e. NONE). If not filtered out, fallthrough to the next filter + // 2. IndexMaintenanceFilter - supports ALL, NONE, and SOME + // In the longer term, we will probably think about deprecating the index maintenance filter. + final FDBStoreTimer timer = state.store.getTimer(); + final IndexPredicate predicate = state.index.getPredicate(); + if (predicate != null) { + final long startTime = timer != null ? System.nanoTime() : 0L; + final boolean useMe = predicate.shouldIndexThisRecord(state.store, savedRecord); + // Note: for now, IndexPredicate will not support filtering of certain index entries + if (timer != null) { + final FDBStoreTimer.Events event = + useMe ? + FDBStoreTimer.Events.USE_INDEX_RECORD_BY_PREDICATE : + FDBStoreTimer.Events.SKIP_INDEX_RECORD_BY_PREDICATE; + timer.recordSinceNanoTime(event, startTime); + } + if (!useMe) { + return IndexMaintenanceFilter.IndexValues.NONE; + } + } + long startTime = System.nanoTime(); + IndexMaintenanceFilter.IndexValues ret = state.filter.maintainIndex(state.index, savedRecord.getRecord()); + if (ret == IndexMaintenanceFilter.IndexValues.NONE && timer != null) { + // events are backward compatible + timer.recordSinceNanoTime(FDBStoreTimer.Events.SKIP_INDEX_RECORD, startTime); + } + return ret; + } + @Nonnull protected List commonKeys(@Nonnull List oldIndexEntries, @Nonnull List newIndexEntries) { diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java index 052276254f..33fdb70bd4 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java @@ -56,6 +56,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexDeferredMaintenanceControl; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter; import com.apple.foundationdb.record.provider.foundationdb.IndexOperation; import com.apple.foundationdb.record.provider.foundationdb.IndexOperationResult; import com.apple.foundationdb.record.provider.foundationdb.IndexScanBounds; @@ -458,9 +459,12 @@ public CompletableFuture update(@Nullable FDBIndexable } @Nonnull - CompletableFuture update(@Nullable FDBIndexableRecord oldRecord, - @Nullable FDBIndexableRecord newRecord, + CompletableFuture update(@Nullable FDBIndexableRecord oldRecordUnfiltered, + @Nullable FDBIndexableRecord newRecordUnfiltered, @Nullable Integer destinationPartitionIdHint) { + FDBIndexableRecord oldRecord = maybeFilterRecord(oldRecordUnfiltered); + FDBIndexableRecord newRecord = maybeFilterRecord(newRecordUnfiltered); + LOG.trace("update oldRecord={}, newRecord={}", oldRecord, newRecord); // Extract information for grouping from old and new records @@ -507,6 +511,19 @@ private CompletableFuture updateRecord( .thenAccept(partitionId -> writeDocument(newRecord, entry, partitionId))); } + @Nullable + public FDBIndexableRecord maybeFilterRecord(FDBIndexableRecord rec) { + if (rec != null) { + final IndexMaintenanceFilter.IndexValues filterType = getFilterTypeForRecord(rec); + if (filterType == IndexMaintenanceFilter.IndexValues.NONE) { + return null; + } else if (filterType == IndexMaintenanceFilter.IndexValues.SOME) { + throw new RecordCoreException("Lucene does not support this kind of filtering"); + } + } + return rec; + } + /** * convenience wrapper that calls {@link #tryDelete(FDBIndexableRecord, Tuple)} only if the index is in * {@code WriteOnly} mode. @@ -782,7 +799,7 @@ public IndexScrubbingTools getIndexScrubbingTools(final IndexScrubbingTools.S final Map options = state.index.getOptions(); if (Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_ENABLED)) || Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_V2_ENABLED))) { - return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager); + return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager, this); } return null; default: diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java index 0b3cf0c00f..eaa4a718d8 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java @@ -67,10 +67,13 @@ public class LuceneIndexScrubbingToolsMissing extends ValueIndexScrubbingToolsMi private final LucenePartitioner partitioner; @Nonnull private final FDBDirectoryManager directoryManager; + @Nonnull + private final LuceneIndexMaintainer indexMaintainer; - public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager) { + public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager, @Nonnull LuceneIndexMaintainer indexMaintainer) { this.partitioner = partitioner; this.directoryManager = directoryManager; + this.indexMaintainer = indexMaintainer; } @@ -100,7 +103,7 @@ public CompletableFuture handleOneItem(final FDBRecordStore store, final } final FDBStoredRecord rec = result.get(); - if (rec == null || !recordTypes.contains(rec.getRecordType())) { + if (!shouldHandleItem(rec)) { return CompletableFuture.completedFuture(null); } @@ -121,6 +124,13 @@ public CompletableFuture handleOneItem(final FDBRecordStore store, final }); } + private boolean shouldHandleItem(FDBStoredRecord rec) { + if (rec == null || !recordTypes.contains(rec.getRecordType())) { + return false; + } + return indexMaintainer.maybeFilterRecord(rec) != null; + } + @SuppressWarnings("PMD.CloseResource") private CompletableFuture> detectMissingIndexKeys(final FDBRecordStore store, FDBStoredRecord rec) { // Generate synthetic record (if applicable) and return the first detected missing (if any). diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java index b6229d28eb..2a8b97c9c6 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java @@ -53,7 +53,6 @@ class LuceneIndexScrubbingTest extends FDBLuceneTestBase { private TestingIndexMaintainerRegistry registry; - private boolean flipBoolean = false; @BeforeEach public void beforeEach() { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java index 57f1b8084b..c253256de3 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java @@ -20,10 +20,13 @@ package com.apple.foundationdb.record.lucene; +import com.apple.foundationdb.record.IndexEntry; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; +import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.TestRecordsGroupedParentChildProto; import com.apple.foundationdb.record.metadata.Index; +import com.apple.foundationdb.record.metadata.IndexPredicate; import com.apple.foundationdb.record.metadata.JoinedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -32,6 +35,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; +import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexScrubber; import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; import com.apple.foundationdb.record.test.TestKeySpace; @@ -316,9 +320,10 @@ private LuceneIndexTestValidator getValidator(final Supplier o @Nonnull static Index addIndex(final boolean isSynthetic, final KeyExpression rootExpression, - final Map options, final RecordMetaDataBuilder metaDataBuilder) { + final Map options, final RecordMetaDataBuilder metaDataBuilder, + @Nullable IndexPredicate predicate) { Index index; - index = new Index("joinNestedConcat", rootExpression, LuceneIndexTypes.LUCENE, options); + index = new Index("joinNestedConcat", rootExpression, LuceneIndexTypes.LUCENE, options, predicate); if (isSynthetic) { final JoinedRecordTypeBuilder joinBuilder = metaDataBuilder.addJoinedRecordType("JoinChildren"); @@ -399,6 +404,28 @@ public void explicitMergeIndex(final FDBRecordContext context, @Nullable FDBStor } } + public long findMissingIndexEntries(final FDBRecordContext context, @Nullable FDBStoreTimer timer) { + FDBRecordStore recordStore = Objects.requireNonNull(schemaSetup.apply(context)); + try (OnlineIndexScrubber indexBuilder = OnlineIndexScrubber.newBuilder() + .setRecordStore(recordStore) + .setIndex(index) + .setTimer(timer) + .build()) { + return indexBuilder.scrubMissingIndexEntries(); + } + } + + public List findAllRecordsByQuery(final FDBRecordContext context, int group) { + LuceneQueryClause search = LuceneQuerySearchClause.MATCH_ALL_DOCS_QUERY; + + FDBRecordStore store = Objects.requireNonNull(schemaSetup.apply(context)); + LuceneScanBounds scanBounds = isGrouped + ? LuceneIndexTestValidator.groupedSortedTextSearch(store, index, search, null, group) + : LuceneIndexTestUtils.fullTextSearch(store, index, search, false); + return store.scanIndex(index, scanBounds, null, ScanProperties.FORWARD_SCAN) + .asList().join(); + } + public Random getRandom() { return random; } @@ -429,6 +456,8 @@ static class Builder { private Index index; @Nullable private RecordMetaData metadata; + @Nullable + IndexPredicate predicate = null; public Builder(final long seed, StoreBuilderSupplier storeBuilderSupplier, TestKeySpacePathManagerExtension pathManager) { @@ -461,6 +490,12 @@ public Builder setPartitionHighWatermark(final int partitionHighWatermark) { return this; } + public Builder setPredicate(@Nullable final IndexPredicate predicate) { + this.predicate = predicate; + metadata = null; + return this; + } + public Builder setTextGeneratorWithNewRandom(final RandomTextGenerator textGenerator) { this.textGenerator = textGenerator.withNewRandom(random); return this; @@ -484,7 +519,7 @@ public LuceneIndexTestDataModel build() { final Map options = getOptions(); final RecordMetaDataBuilder metaDataBuilder = LuceneIndexTestDataModel.createBaseMetaDataBuilder(); final KeyExpression rootExpression = LuceneIndexTestDataModel.createRootExpression(isGrouped, isSynthetic); - this.index = LuceneIndexTestDataModel.addIndex(isSynthetic, rootExpression, options, metaDataBuilder); + this.index = LuceneIndexTestDataModel.addIndex(isSynthetic, rootExpression, options, metaDataBuilder, predicate); this.metadata = metaDataBuilder.build(); } final Function schemaSetup = context -> { diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java index d1fbb6b933..d37eaa8ae6 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneOnlineIndexingTest.java @@ -46,6 +46,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainer; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactory; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter; import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer; import com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils; import com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage; @@ -55,6 +56,7 @@ import com.apple.foundationdb.record.util.pair.Pair; import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.BooleanSource; import com.apple.test.RandomSeedSource; import com.apple.test.RandomizedTestUtils; import com.google.auto.service.AutoService; @@ -106,6 +108,7 @@ import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.MAP_DOC; import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -633,7 +636,6 @@ void luceneOnlineIndexingTestMulti() throws IOException { } } - protected void openRecordStore(FDBRecordContext context, FDBRecordStoreTestBase.RecordMetaDataHook hook) { RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsTextProto.getDescriptor()); metaDataBuilder.getRecordType(COMPLEX_DOC).setPrimaryKey(concatenateFields("group", "doc_id")); @@ -644,6 +646,23 @@ protected void openRecordStore(FDBRecordContext context, FDBRecordStoreTestBase. setupPlanner(null); } + protected void openRecordStoreWithFilter(FDBRecordContext context, FDBRecordStoreTestBase.RecordMetaDataHook hook, boolean filterOut) { + RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecordsTextProto.getDescriptor()); + metaDataBuilder.getRecordType(COMPLEX_DOC).setPrimaryKey(concatenateFields("group", "doc_id")); + hook.apply(metaDataBuilder); + final FDBRecordStore.Builder builder = getStoreBuilder(context, metaDataBuilder.getRecordMetaData()) + .setSerializer(TextIndexTestUtils.COMPRESSING_SERIALIZER); + if (filterOut) { + recordStore = builder + .setIndexMaintenanceFilter((i, r) -> IndexMaintenanceFilter.IndexValues.NONE) + .createOrOpen(); + } else { + recordStore = builder + .createOrOpen(); + } + setupPlanner(null); + } + @ParameterizedTest @ValueSource(ints = {1, 2, 3}) void luceneOnlineIndexingTestGroupingKeys(int groupingCount) { @@ -760,6 +779,43 @@ void luceneOnlineIndexingTestGroupingKeysBackgroundMerge(int groupingCount) thro assertTrue(newLength < oldLength); } + @ParameterizedTest + @BooleanSource + void luceneOnlineIndexingTestNoMergeIfFilteredOutRecords(boolean filterOut) throws IOException { + Index index = new Index( + "Map_with_auto_complete$entry-value", + new GroupingKeyExpression(field("entry", + KeyExpression.FanType.FanOut).nest(concat(LuceneIndexTestUtils.keys)), 3), + LuceneIndexTypes.LUCENE, + ImmutableMap.of()); + + RecordMetaDataHook hook = metaDataBuilder -> { + metaDataBuilder.removeIndex(TextIndexTestUtils.SIMPLE_DEFAULT_NAME); + TextIndexTestUtils.addRecordTypePrefix(metaDataBuilder); + metaDataBuilder.addIndex(MAP_DOC, index); + }; + int group = 3; + + // write/overwrite records + boolean needMerge = false; + for (int iLast = 60; iLast > 40; iLast --) { + try (FDBRecordContext context = openContext()) { + openRecordStoreWithFilter(context, hook, filterOut); + for (int i = 0; i < iLast; i++) { + recordStore.saveRecord(multiEntryMapDoc(77L * i, ENGINEER_JOKE + iLast, group)); + } + final Set indexSet = recordStore.getIndexDeferredMaintenanceControl().getMergeRequiredIndexes(); + if (indexSet != null && !indexSet.isEmpty()) { + assertEquals(1, indexSet.size()); + assertEquals(indexSet.stream().findFirst().get().getName(), index.getName()); + needMerge = true; + } + commit(context); + } + } + assertNotEquals(needMerge, filterOut); + } + private TestRecordsTextProto.MapDocument multiEntryMapDoc(long id, String text, int group) { assertTrue(group < 4); String text2 = "Text 2, and " + (id % 2); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneScanAllEntriesTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneScanAllEntriesTest.java index 9f5b4f26fc..cf682dde32 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneScanAllEntriesTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneScanAllEntriesTest.java @@ -23,30 +23,51 @@ import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.IndexEntry; import com.apple.foundationdb.record.IsolationLevel; +import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordMetaDataProvider; import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.TestRecordsGroupedParentChildProto; import com.apple.foundationdb.record.cursors.AutoContinuingCursor; +import com.apple.foundationdb.record.metadata.IndexPredicate; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabaseRunner; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreConcurrentTestBase; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; +import com.apple.foundationdb.record.provider.foundationdb.IndexMaintenanceFilter; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath; +import com.apple.foundationdb.record.query.expressions.Comparisons; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate; +import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.tuple.Tuple; import com.apple.test.BooleanSource; import com.apple.test.Tags; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import javax.annotation.Nonnull; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.LongPredicate; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.apple.foundationdb.record.lucene.LuceneIndexTestDataModel.CHILD_SEARCH_TERM; import static com.apple.foundationdb.record.lucene.LuceneIndexTestDataModel.PARENT_SEARCH_TERM; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test for Lucene index scanning where the query contains "*:*" term matching all documents. @@ -211,4 +232,309 @@ private void assertIndexEntryPrimaryKeyTuples(Set primaryKeys, RecordCurs assertEquals(primaryKeys, indexEntries.stream().map(IndexEntry::getPrimaryKey).collect(Collectors.toSet())); } + + @ParameterizedTest + @BooleanSource + public void indexScanWithEvenRecNoFilterTest(boolean isGrouped) { + final long seed = 9876543L; + final boolean isSynthetic = false; + + final LuceneIndexTestDataModel dataModel = + new LuceneIndexTestDataModel.Builder(seed, this::getStoreBuilderFilterOddRecNo, pathManager) + .setIsGrouped(isGrouped) + .setIsSynthetic(isSynthetic) + .setPrimaryKeySegmentIndexEnabled(true) + .setPartitionHighWatermark(10) + .build(); + + try (FDBRecordContext context = openContext()) { + // Save 10 records - only even recNo values should be indexed + dataModel.saveRecords(10, context, 2); + commit(context); + } + + try (FDBRecordContext context = openContext()) { + List indexEntries = dataModel.findAllRecordsByQuery(context, 2); + // We expect 5 records with even recNo values to be indexed + assertEquals(5, indexEntries.size(), "Should have indexed only records with even recNo"); + verifyByRecNo(indexEntries, dataModel.createOrOpenRecordStore(context), recNo -> (recNo & 1) == 0); + } + + try (FDBRecordContext context = openContext()) { + // overwrite records + dataModel.saveRecords(10, context, 2); + commit(context); + } + + try (FDBRecordContext context = openContext()) { + // The same filter should apply to index scrubbing - else "missing" index entries will be detected + final long missingIndexEntries = dataModel.findMissingIndexEntries(context, null); + assertEquals(0, missingIndexEntries); + } + } + + private void verifyByRecNo(final List indexEntries, final FDBRecordStore store, LongPredicate verifier) { + // Verify that all indexed records have even recNo + for (IndexEntry entry : indexEntries) { + Tuple primaryKey = entry.getPrimaryKey(); + // The recNo is part of the primary key - need to extract and verify it's even + // For grouped records, structure is (group, recNo) or similar + // For non-synthetic parent records, the recNo is in the tuple + FDBStoredRecord storedRecord = store.loadRecord(primaryKey); + assertNotNull(storedRecord, "Record should exist"); + Message message = storedRecord.getRecord(); + Descriptors.FieldDescriptor recNoField = + message.getDescriptorForType().findFieldByName("rec_no"); + long recNo = (long) message.getField(recNoField); + assertTrue(verifier.test(recNo), "Unexpected recNo was found. recNo: " + recNo); + } + } + + @Nonnull + private FDBRecordStore.Builder getStoreBuilderFilterOddRecNo(@Nonnull FDBRecordContext context, + @Nonnull RecordMetaDataProvider metaData, + @Nonnull final KeySpacePath path) { + // Create an index maintenance filter that only indexes records with even recNo + IndexMaintenanceFilter evenRecNoFilter = (index, rec) -> { + Descriptors.FieldDescriptor recNoField = + rec.getDescriptorForType().findFieldByName("rec_no"); + if (recNoField != null && rec.hasField(recNoField)) { + long recNo = (long) rec.getField(recNoField); + if (recNo % 2 == 0) { + return IndexMaintenanceFilter.IndexValues.ALL; + } + } + return IndexMaintenanceFilter.IndexValues.NONE; + }; + + return getStoreBuilder(context, metaData, path) + .setIndexMaintenanceFilter(evenRecNoFilter); + } + + @ParameterizedTest + @BooleanSource + public void indexScanWithRecNoIndexPredicateTest(boolean isGrouped) { + final long seed = 5432198L; + + // Create an index predicate that only indexes records with even recNo + // We'll create a predicate: rec_no % 2 == 0 + // Since there's no modulo operator, we'll use an OR of specific even values + final Type.Record recordType = Type.Record.fromDescriptor( + TestRecordsGroupedParentChildProto.MyParentRecord.getDescriptor()); + final QuantifiedObjectValue recordValue = QuantifiedObjectValue.of(Quantifier.current(), recordType); + final FieldValue recNoField = FieldValue.ofFieldName(recordValue, "rec_no"); + + // Create predicate for rec_no > 1006 + final QueryPredicate filterPredicate = new ValuePredicate(recNoField, new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 1006L)); + + // Convert to IndexPredicate + final IndexPredicate indexPredicate = IndexPredicate.fromQueryPredicate(filterPredicate); + + // Build the data model using the Builder + final LuceneIndexTestDataModel dataModel = new LuceneIndexTestDataModel.Builder(seed, this::getStoreBuilder, pathManager) + .setIsGrouped(isGrouped) + .setIsSynthetic(false) + .setPrimaryKeySegmentIndexEnabled(true) + .setPartitionHighWatermark(10) + .setPredicate(indexPredicate) + .build(); + + try (FDBRecordContext context = openContext()) { + dataModel.saveRecords(10, context, 2); + commit(context); + } + + try (FDBRecordContext context = openContext()) { + List indexEntries = dataModel.findAllRecordsByQuery(context, 2); + + // We expect 5 records with recNo > 1006 values to be indexed + assertEquals(5, indexEntries.size(), "Should have indexed only records with even recNo"); + verifyByRecNo(indexEntries, dataModel.createOrOpenRecordStore(context), recNo -> recNo > 1006); + } + + try (FDBRecordContext context = openContext()) { + // The same filter should apply to index scrubbing - else "missing" index entries will be detected + final long missingIndexEntries = dataModel.findMissingIndexEntries(context, null); + assertEquals(0, missingIndexEntries); + } + } + + @ParameterizedTest + @BooleanSource + public void indexScanWithSomeFilterThrowsExceptionTest(boolean isGrouped) { + final long seed = 1234567L; + + final LuceneIndexTestDataModel dataModel = + new LuceneIndexTestDataModel.Builder(seed, this::getStoreBuilderWithSomeFilter, pathManager) + .setIsGrouped(isGrouped) + .setIsSynthetic(false) + .setPrimaryKeySegmentIndexEnabled(true) + .setPartitionHighWatermark(10) + .build(); + + // Attempt to save records - should throw RecordCoreException because SOME is not supported + RecordCoreException exception = Assertions.assertThrows(RecordCoreException.class, () -> { + try (FDBRecordContext context = openContext()) { + // Try to save a single record with the SOME filter active + // This should trigger the exception during index maintenance + dataModel.saveRecords(1, context); + commit(context); + } + }); + + // Verify the exception message + assertEquals("Lucene does not support this kind of filtering", exception.getMessage()); + } + + @Nonnull + private FDBRecordStore.Builder getStoreBuilderWithSomeFilter(@Nonnull FDBRecordContext context, + @Nonnull RecordMetaDataProvider metaData, + @Nonnull final KeySpacePath path) { + // Create an index maintenance filter that returns SOME + // This should trigger an exception since Lucene doesn't support partial indexing + IndexMaintenanceFilter someFilter = (index, rec) -> IndexMaintenanceFilter.IndexValues.SOME; + + return getStoreBuilder(context, metaData, path) + .setIndexMaintenanceFilter(someFilter); + } + + @ParameterizedTest + @BooleanSource + public void indexScanWithFailingFilterThrowsExceptionTest(boolean isGrouped) { + final long seed = 7654321L; + + final LuceneIndexTestDataModel dataModel = + new LuceneIndexTestDataModel.Builder(seed, this::getStoreBuilderWithFailingFilterFor1002L, pathManager) + .setIsGrouped(isGrouped) + .setIsSynthetic(false) + .setPrimaryKeySegmentIndexEnabled(true) + .setPartitionHighWatermark(10) + .build(); + + // Attempt to save records - should throw RecordCoreException from the filter for recNo 1002L + try (FDBRecordContext context = openContext()) { + RecordCoreException exception = Assertions.assertThrows(RecordCoreException.class, () -> dataModel.saveRecords(1, context, 2)); + + Assertions.assertTrue(exception.getMessage().startsWith("Filter failed for recNo:"), + "Exception message should indicate filter failure"); + Assertions.assertEquals(1002L, exception.getLogInfo().get("rec_no"), + "Exception should log the rec_no that caused the failure"); + Assertions.assertEquals(dataModel.index.getName(), exception.getLogInfo().get("index"), + "Exception should log the index name"); + } + } + + @Nonnull + private FDBRecordStore.Builder getStoreBuilderWithFailingFilterFor1002L(@Nonnull FDBRecordContext context, + @Nonnull RecordMetaDataProvider metaData, + @Nonnull final KeySpacePath path) { + // Create an index maintenance filter that throws an exception for specific record with recNo 1002L + // This can be used to tests error handling when the filter itself fails + IndexMaintenanceFilter failingFilter = (index, rec) -> { + Descriptors.FieldDescriptor recNoField = + rec.getDescriptorForType().findFieldByName("rec_no"); + if (recNoField != null && rec.hasField(recNoField)) { + long recNo = (long) rec.getField(recNoField); + if (recNo == 1002L) { + throw new RecordCoreException("Filter failed for recNo: " + recNo) + .addLogInfo("rec_no", recNo) + .addLogInfo("index", index.getName()); + } + return IndexMaintenanceFilter.IndexValues.ALL; + } + return IndexMaintenanceFilter.IndexValues.NONE; + }; + + return getStoreBuilder(context, metaData, path) + .setIndexMaintenanceFilter(failingFilter); + } + + @ParameterizedTest + @BooleanSource + public void indexScanWithEvenRecNoFilterSyntheticTest(boolean isGrouped) { + final long seed = 777L; + + final LuceneIndexTestDataModel dataModel = + new LuceneIndexTestDataModel.Builder(seed, this::getStoreBuilderFilterOddRecNoSynthetic, pathManager) + .setIsGrouped(isGrouped) + .setIsSynthetic(true) + .setPrimaryKeySegmentIndexEnabled(true) + .setPartitionHighWatermark(10) + .build(); + + try (FDBRecordContext context = openContext()) { + dataModel.saveRecords(10, context, 2); + commit(context); + } + + try (FDBRecordContext context = openContext()) { + List indexEntries = dataModel.findAllRecordsByQuery(context, 2); + // We expect 5 synthetic records with even parent rec_no values to be indexed + assertEquals(5, indexEntries.size(), "Should have indexed only synthetic records with even parent rec_no"); + verifyEvenRecNoOnlySynthetic(indexEntries, dataModel.createOrOpenRecordStore(context)); + } + + try (FDBRecordContext context = openContext()) { + // Overwrite records to test update path + dataModel.saveRecords(10, context, 2); + commit(context); + } + + try (FDBRecordContext context = openContext()) { + // The same filter should apply to index scrubbing - else "missing" index entries will be detected + final long missingIndexEntries = dataModel.findMissingIndexEntries(context, null); + assertEquals(0, missingIndexEntries); + } + } + + @Nonnull + private FDBRecordStore.Builder getStoreBuilderFilterOddRecNoSynthetic(@Nonnull FDBRecordContext context, + @Nonnull RecordMetaDataProvider metaData, + @Nonnull final KeySpacePath path) { + // Create an index maintenance filter that only indexes synthetic records with even parent rec_no + IndexMaintenanceFilter evenRecNoFilter = (index, rec) -> { + // For synthetic records, we need to access the parent constituent + // The synthetic record has structure: { parent: MyParentRecord, child: MyChildRecord } + Descriptors.FieldDescriptor parentField = + rec.getDescriptorForType().findFieldByName("parent"); + if (parentField != null && rec.hasField(parentField)) { + Message parentMessage = (Message) rec.getField(parentField); + Descriptors.FieldDescriptor recNoField = + parentMessage.getDescriptorForType().findFieldByName("rec_no"); + if (recNoField != null && parentMessage.hasField(recNoField)) { + long recNo = (long) parentMessage.getField(recNoField); + if (recNo % 2 == 0) { + return IndexMaintenanceFilter.IndexValues.ALL; + } + } + } + return IndexMaintenanceFilter.IndexValues.NONE; + }; + + return getStoreBuilder(context, metaData, path) + .setIndexMaintenanceFilter(evenRecNoFilter); + } + + @SuppressWarnings("unchecked") + private void verifyEvenRecNoOnlySynthetic(final List indexEntries, FDBRecordStore store) { + // Verify that all indexed synthetic records have even rec_no at parent + for (IndexEntry entry : indexEntries) { + Tuple syntheticPrimaryKey = entry.getPrimaryKey(); + + // Extract parent primary key from synthetic primary key + List parentKeyItems = (List) syntheticPrimaryKey.get(1); + Tuple parentPrimaryKey = Tuple.fromList(parentKeyItems); + + // Load the parent record using the extracted parent primary key + FDBStoredRecord storedRecord = store.loadRecord(parentPrimaryKey); + assertNotNull(storedRecord, "Parent record should exist"); + Message message = storedRecord.getRecord(); + + Descriptors.FieldDescriptor recNoField = + message.getDescriptorForType().findFieldByName("rec_no"); + long recNo = (long) message.getField(recNoField); + + assertEquals(0, recNo % 2, "All indexed synthetic records should have even parent rec_no, but found: " + recNo); + } + } }