Introduce KeySpacePath.importData to import previously exported data

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java

@@ -182,6 +182,8 @@ public enum Events implements StoreTimer.Event {
         RANGE_SET_CONTAINS("range set contains key"),
         /** The amount of time checking if a {@link com.google.common.collect.RangeSet} is empty. */
         RANGE_SET_IS_EMPTY("range set is empty"),
+        /** The amount of time importing a single KeyValue into a path. */
+        IMPORT_DATA("import KeyValue"),
 
         /** The amount of time spent clearing the space taken by an index that has been removed from the meta-data. */
         REMOVE_FORMER_INDEX("remove former index"),

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java

@@ -24,10 +24,13 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.RecordCoreArgumentException;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
+import com.apple.foundationdb.tuple.ByteArrayUtil2;
 import com.apple.foundationdb.tuple.Tuple;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Objects;
 
 /**
  * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}.
@@ -67,4 +70,22 @@ public Tuple getRemainder() {
         return remainder;
     }
 
+    @Override
+    public boolean equals(final Object o) {
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final DataInKeySpacePath that = (DataInKeySpacePath)o;
+        return Objects.equals(path, that.path) && Objects.equals(remainder, that.remainder) && Arrays.equals(value, that.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(path, remainder, Arrays.hashCode(value));
+    }
+
+    @Override
+    public String toString() {
+        return path + "+" + remainder + "->" + ByteArrayUtil2.loggable(value);
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java

@@ -410,4 +410,26 @@ default RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext
                                                            @Nonnull ScanProperties scanProperties) {
         throw new UnsupportedOperationException("exportAllData is not supported");
     }
+
+    /**
+     * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}.
+     * <p>
+     * This will validate that any data provided in {@code dataToImport} has a path that should be in this path,
+     * or one of the sub-directories, if not, the future will complete exceptionally with
+     * {@link RecordCoreIllegalImportDataException}.
+     * If there is any data already existing under this path, the new data will overwrite if the keys are the same.
+     * This will use the logical values in the {@link DataInKeySpacePath#getPath()} and
+     * {@link DataInKeySpacePath#getRemainder()} to determine the key, rather
+     * than the raw key, meaning that this will work even if the data was exported from a different cluster.
+     * Note, this will not correct for any cluster-specific data, other than {@link DirectoryLayerDirectory} data;
+     * for example, if you have versionstamps, that data will not align on the destination.
+     * </p>
+     * @param context the transaction context in which to save the data
+     * @param dataToImport the data to be saved to the database
+     * @return a future to be completed once all data has been important.
+     */
+    @API(API.Status.EXPERIMENTAL)
+    @Nonnull
+    CompletableFuture<Void> importData(@Nonnull FDBRecordContext context,
+                                       @Nonnull Iterable<DataInKeySpacePath> dataToImport);
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java

@@ -29,6 +29,7 @@
 import com.apple.foundationdb.record.cursors.LazyCursor;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext;
+import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer;
 import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor;
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
@@ -316,6 +317,51 @@ public RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext
                     1);
     }
 
+    @Nonnull
+    @Override
+    public CompletableFuture<Void> importData(@Nonnull FDBRecordContext context,
+                                              @Nonnull Iterable<DataInKeySpacePath> dataToImport) {
+        return toTupleAsync(context).thenCompose(targetTuple -> {
+            // We use a mapPipelined here to help control the rate of insertions into the directory layer if those
+            // are happening.
+            // DirectoryLayer operations are done in a separate transaction for two reasons:
+            // 1. To reduce conflicts
+            // 2. So that it can immediately be cached without having to do it in a post-commit hook, which can make
+            //    things complicated
+            // So if we just spun off a future for every data item, they would conflict like crazy, and retrying all
+            // of those conflicts would cause this future to take way longer than if we pipeline.
+            // This shouldn't make much of a difference in the general case because almost all the directory layer
+            // lookups should be from cache.
+            final RecordCursor<Void> insertionWork = RecordCursor.fromIterator(dataToImport.iterator())
+                    .mapPipelined(dataItem ->
+                            dataItem.getPath().toTupleAsync(context).thenAccept(itemPathTuple -> {
+                                // Validate that this data belongs under this path
+                                if (!TupleHelpers.isPrefix(targetTuple, itemPathTuple)) {
+                                    throw new RecordCoreIllegalImportDataException(
+                                            "Data item path does not belong under target path",
+                                            "target", targetTuple,
+                                            "item", itemPathTuple);
+                                }
+
+                                // Reconstruct the key using the path and remainder
+                                Tuple keyTuple = itemPathTuple;
+                                if (dataItem.getRemainder() != null) {
+                                    keyTuple = keyTuple.addAll(dataItem.getRemainder());
+                                }
+
+                                // Store the data
+                                byte[] keyBytes = keyTuple.pack();
+                                byte[] valueBytes = dataItem.getValue();
+                                context.ensureActive().set(keyBytes, valueBytes);
+                            }),
+                            1);
+            // Use forEach to force consuming the entire cursor, which will cause the inserts to happen
+            final CompletableFuture<Void> allInsertions = insertionWork.forEach(vignore -> { })
+                    .whenComplete((vignore, e) -> insertionWork.close());
+            return context.instrument(FDBStoreTimer.Events.IMPORT_DATA, allInsertions);
+        });
+    }
+
     /**
      * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates.
      */

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java

@@ -202,4 +202,11 @@ public RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext
                                                           @Nonnull ScanProperties scanProperties) {
         return inner.exportAllData(context, continuation, scanProperties);
     }
+
+    @Nonnull
+    @Override
+    public CompletableFuture<Void> importData(@Nonnull FDBRecordContext context,
+                                              @Nonnull Iterable<DataInKeySpacePath> dataToImport) {
+        return inner.importData(context, dataToImport);
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java

@@ -0,0 +1,36 @@
+/*
+ * RecordCoreIllegalImportDataException.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.provider.foundationdb.keyspace;
+
+import com.apple.foundationdb.record.RecordCoreArgumentException;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Thrown if the data being imported into {@link KeySpacePath#importData} does not belong in that path.
+ */
+public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException {
+    private static final long serialVersionUID = 1L;
+
+    public RecordCoreIllegalImportDataException(@Nonnull final String msg, @Nonnull final Object... keyValue) {
+        super(msg, keyValue);
+    }
+}

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java

@@ -31,9 +31,11 @@
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Tests for {@link DataInKeySpacePath}.
@@ -118,4 +120,105 @@
         assertThrows(RecordCoreArgumentException.class,
                 () -> new DataInKeySpacePath(testPath, null, valueBytes));
     }
+
+    @Test
+    void testEquals() {
+        KeySpace root = new KeySpace(
+                new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString()));
+
+        KeySpacePath testPath = root.path("test");
+        Tuple remainder1 = Tuple.from("key1", "key2");
+        byte[] value1 = Tuple.from("value1").pack();
+
+        DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder1, value1);
+        DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder1, value1);
+
+        // Reflexive: object equals itself
+        assertEquals(data1, data1);
+
+        // Symmetric: a.equals(b) implies b.equals(a)
+        assertEquals(data1, data2);
+        assertEquals(data2, data1);
+
+        // Test with different remainder
+        Tuple remainder2 = Tuple.from("different", "key");
+        DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, remainder2, value1);
+        assertNotEquals(data1, data3);
+
+        // Test with different value
+        byte[] value2 = Tuple.from("value2").pack();
+        DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, remainder1, value2);
+        assertNotEquals(data1, data4);
+
+        // Test with different path
+        KeySpace root2 = new KeySpace(
+                new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString()));
+        KeySpacePath testPath2 = root2.path("test");
+        DataInKeySpacePath data5 = new DataInKeySpacePath(testPath2, remainder1, value1);
+        assertNotEquals(data1, data5);
+
+        // Test with null remainder
+        DataInKeySpacePath data6 = new DataInKeySpacePath(testPath, null, value1);
+        DataInKeySpacePath data7 = new DataInKeySpacePath(testPath, null, value1);
+        assertEquals(data6, data7);
+        assertNotEquals(data1, data6);
+
+        // Test with null object
+        assertNotEquals(data1, null);
+
+        // Test with different class
+        assertNotEquals(data1, "not a DataInKeySpacePath");
+    }
+
+    @Test
+    void testHashCode() {
+        KeySpace root = new KeySpace(
+                new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString()));
+
+        KeySpacePath testPath = root.path("test");
+        Tuple remainder = Tuple.from("key1", "key2");
+        byte[] value = Tuple.from("value1").pack();
+
+        DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder, value);
+        DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder, value);
+
+        // Equal objects must have equal hash codes
+        assertEquals(data1.hashCode(), data2.hashCode());
+
+        // Test with null remainder
+        DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, null, value);
+        DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, null, value);
+        assertEquals(data3.hashCode(), data4.hashCode());
+
+        // Different objects should generally have different hash codes (not required, but good practice)
+        Tuple remainder2 = Tuple.from("different", "key");
+        DataInKeySpacePath data5 = new DataInKeySpacePath(testPath, remainder2, value);
+        assertNotEquals(data1.hashCode(), data5.hashCode());
+    }
+
+    @Test
+    void testToString() {
+        final String rootUuid = UUID.randomUUID().toString();
+        KeySpace root = new KeySpace(
+                new KeySpaceDirectory("test", KeyType.STRING, rootUuid));
+
+        KeySpacePath testPath = root.path("test");
+        Tuple remainder = Tuple.from("key1", "key2");
+        byte[] value = Tuple.from("value1").pack();
+
+        DataInKeySpacePath data = new DataInKeySpacePath(testPath, remainder, value);
+
+        String result = data.toString();
+
+        // Verify the string contains expected components
+        assertTrue(result.contains(rootUuid));
+        assertTrue(result.contains("test"));
+        assertTrue(result.contains("key1"), "toString should contain remainder elements");
+        assertTrue(result.contains("key2"), "toString should contain remainder elements");
+
+        // Test with null remainder
+        DataInKeySpacePath dataWithNullRemainder = new DataInKeySpacePath(testPath, null, value);
+        String resultWithNull = dataWithNullRemainder.toString();
+        assertTrue(resultWithNull.contains("null"), "toString should contain 'null' for null remainder");
+    }
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java

@@ -49,7 +49,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -69,7 +68,7 @@ class KeySpacePathDataExportTest {
     final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension();
 
     @Test
-    void exportAllDataFromSimplePath() throws ExecutionException, InterruptedException {
+    void exportAllDataFromSimplePath() {
         KeySpace root = new KeySpace(
                 new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString())
                         .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG)));
@@ -83,13 +82,14 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti
 
             // Add data at different levels
             for (int i = 0; i < 5; i++) {
-                Tuple key = basePath.add("level1", (long) i).toTuple(context);
+                final KeySpacePath path = basePath.add("level1", (long)i);
+                Tuple key = path.toTuple(context);
                 tr.set(key.pack(), Tuple.from("value" + i).pack());
 
                 // Add some sub-data under each key
                 for (int j = 0; j < 3; j++) {
-                    Tuple subKey = key.add("sub" + j);
-                    tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack());
+                    tr.set(path.toSubspace(context).pack(Tuple.from("sub" + j)),
+                            Tuple.from("subvalue" + i + "_" + j).pack());
                 }
             }
             context.commit();
@@ -103,16 +103,19 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti
             // Should have 5 main entries + 15 sub-entries = 20 total
             assertEquals(20, allData.size());
 
+            assertThat(allData)
+                    .allSatisfy(data ->
+                            assertThat(data.getPath().getDirectoryName()).isEqualTo("level1"));
+
             // Verify the data is sorted by key
-            for (int i = 1; i < allData.size(); i++) {
-                assertTrue(getKey(allData.get(i - 1), context).compareTo(getKey(allData.get(i), context)) < 0);
-            }
+            assertThat(allData.stream().map(data -> getKey(data, context)).collect(Collectors.toList()))
+                    .isSorted();
         }
     }
 
     // `toTuple` does not include the remainder, I'm not sure if that is intentional, or an oversight.
-    private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) throws ExecutionException, InterruptedException {
-        final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).get();
+    private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) {
+        final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).join();
         if (dataInKeySpacePath.getRemainder() != null) {
             return resolvedKeySpacePath.toTuple().addAll(dataInKeySpacePath.getRemainder());
         } else {
@@ -524,9 +527,7 @@ private static void exportWithContinuations(final KeySpacePath pathToExport,
                 final RecordCursor<DataInKeySpacePath> cursor = pathToExport.exportAllData(context, continuation.toBytes(),
                         scanProperties);
                 final AtomicReference<RecordCursorResult<Tuple>> tupleResult = new AtomicReference<>();
-                final List<Tuple> batch = cursor.map(dataInPath -> {
-                    return Tuple.fromBytes(dataInPath.getValue());
-                }).asList(tupleResult).join();
+                final List<Tuple> batch = cursor.map(dataInPath -> Tuple.fromBytes(dataInPath.getValue())).asList(tupleResult).join();
                 actual.add(batch);
                 continuation = tupleResult.get().getContinuation();
             }
@@ -578,7 +579,7 @@ void exportAllDataThroughKeySpacePathWrapper() {
     }
 
     @Test
-    void exportAllDataThroughKeySpacePathWrapperResolvedPaths() {
+    void exportAllDataThroughKeySpacePathWrapperRemainders() {
         final FDBDatabase database = dbExtension.getDatabase();
         final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database);