Skip to content

Commit 7854ffc

Browse files
hatyohatyo
committed
Merge remote-tracking branch 'upstream/main' into vector-sql-type
2 parents badd376 + 7b63113 commit 7854ffc

25 files changed

+529
-176
lines changed

‎docs/sphinx/source/ReleaseNotes.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,44 @@ As the [versioning guide](Versioning.md) details, it cannot always be determined
77

88
## 4.8
99

10+
### 4.8.5.0
11+
12+
13+
14+
**[Full Changelog (4.8.4.0...4.8.5.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.8.4.0...4.8.5.0)**
15+
16+
#### Mixed Mode Test Results
17+
18+
Mixed mode testing run against the following previous versions:
19+
20+
❌`4.6.2.0`, ❌`4.6.3.0`, ❌`4.6.4.0`, ❌`4.6.5.0`, ❌`4.7.1.0`, ❌`4.7.2.0`, ✅`4.7.3.0`, ✅`4.8.1.0`, ✅`4.8.2.0`, ✅`4.8.3.0`
21+
22+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/18949783608)
23+
24+
25+
26+
### 4.8.4.0
27+
28+
<h4> New Features </h4>
29+
30+
* Protobuf compliant translation for supporting richer identifiers for tables and columns - [PR #3696](https://github.com/FoundationDB/fdb-record-layer/pull/3696)
31+
<h4> Performance Improvements </h4>
32+
33+
* Reactivate amortization of plan generation of functions. - [PR #3595](https://github.com/FoundationDB/fdb-record-layer/pull/3595)
34+
35+
36+
**[Full Changelog (4.8.3.0...4.8.4.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.8.3.0...4.8.4.0)**
37+
38+
#### Mixed Mode Test Results
39+
40+
Mixed mode testing run against the following previous versions:
41+
42+
❌`4.6.2.0`, ❌`4.6.3.0`, ❌`4.6.4.0`, ❌`4.6.5.0`, ❌`4.7.1.0`, ❌`4.7.2.0`, ✅`4.7.3.0`, ✅`4.8.1.0`, ✅`4.8.2.0`, ✅`4.8.3.0`
43+
44+
[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/18946501767)
45+
46+
47+
1048
### 4.8.3.0
1149

1250
<h4> New Features </h4>

‎fdb-relational-core/src/main/antlr/RelationalParser.g4‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -879,7 +879,7 @@ recordConstructorForInlineTable
879879
;
880880

881881
recordConstructor
882-
: ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithName /* this can be removed */ | expressionWithOptionalName (',' expressionWithOptionalName)*) ')'
882+
: ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithOptionalName (',' expressionWithOptionalName)*) ')'
883883
;
884884

885885
ofTypeClause
@@ -913,10 +913,6 @@ expressionOrDefault
913913
: expression | DEFAULT
914914
;
915915

916-
expressionWithName
917-
: expression AS uid
918-
;
919-
920916
expressionWithOptionalName
921917
: expression (AS uid)?
922918
;

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/ddl/RecordLayerCatalogQueryFactory.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import com.apple.foundationdb.relational.api.metadata.SchemaTemplate;
3838
import com.apple.foundationdb.relational.recordlayer.ArrayRow;
3939
import com.apple.foundationdb.relational.recordlayer.IteratorResultSet;
40+
import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils;
4041

4142
import javax.annotation.Nonnull;
4243
import java.net.URI;
@@ -64,7 +65,7 @@ public Type getResultSetMetadata() {
6465
public RelationalResultSet executeAction(Transaction txn) throws RelationalException {
6566
final Schema schema = catalog.loadSchema(txn, dbId, schemaId);
6667

67-
final List<String> tableNames = schema.getTables().stream().map(Metadata::getName)
68+
final List<String> tableNames = schema.getTables().stream().map(Metadata::getName).map(DataTypeUtils::toUserIdentifier)
6869
.collect(Collectors.toList());
6970

7071
final List<String> indexNames = schema.getTables().stream().flatMap(t -> t.getIndexes().stream()).map(Metadata::getName)

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/DataTypeUtils.java‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@
2121
package com.apple.foundationdb.relational.recordlayer.metadata;
2222

2323
import com.apple.foundationdb.annotation.API;
24-
2524
import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
2625
import com.apple.foundationdb.relational.api.exceptions.ErrorCode;
2726
import com.apple.foundationdb.relational.api.metadata.DataType;
2827
import com.apple.foundationdb.relational.util.Assert;
2928
import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings;
30-
3129
import com.google.common.collect.BiMap;
3230
import com.google.common.collect.HashBiMap;
3331

@@ -41,6 +39,10 @@
4139
@API(API.Status.EXPERIMENTAL)
4240
public class DataTypeUtils {
4341

42+
private static final String DOUBLE_UNDERSCORE_ESCAPE = "__0";
43+
private static final String DOLLAR_ESCAPE = "__1";
44+
private static final String DOT_ESCAPE = "__2";
45+
4446
@Nonnull
4547
private static final BiMap<DataType, Type> primitivesMap;
4648

@@ -55,6 +57,20 @@ public class DataTypeUtils {
5557
@SpotBugsSuppressWarnings(value = "NP_NONNULL_RETURN_VIOLATION", justification = "should never happen, there is failUnchecked directly before that.")
5658
@Nonnull
5759
public static DataType toRelationalType(@Nonnull final Type type) {
60+
return toRelationalType(type, false);
61+
}
62+
63+
/**
64+
* Converts a Record Layer {@link Type} into a Relational {@link DataType}.
65+
*
66+
* Note: This method is expensive, use with care, i.e. try to cache its result as much as possible.
67+
*
68+
* @param type The Relational data type.
69+
* @return The corresponding Record Layer type.
70+
*/
71+
@SpotBugsSuppressWarnings(value = "NP_NONNULL_RETURN_VIOLATION", justification = "should never happen, there is failUnchecked directly before that.")
72+
@Nonnull
73+
public static DataType toRelationalType(@Nonnull final Type type, boolean toUserIdentifier) {
5874
if (primitivesMap.containsValue(type)) {
5975
return primitivesMap.inverse().get(type);
6076
}
@@ -75,32 +91,44 @@ public static DataType toRelationalType(@Nonnull final Type type) {
7591
switch (typeCode) {
7692
case RECORD:
7793
final var record = (Type.Record) type;
78-
final var columns = record.getFields().stream().map(field -> DataType.StructType.Field.from(field.getFieldName(), toRelationalType(field.getFieldType()), field.getFieldIndex())).collect(Collectors.toList());
79-
return DataType.StructType.from(record.getName() == null ? toProtoBufCompliantName(UUID.randomUUID().toString()) : record.getName(), columns, record.isNullable());
94+
final var columns = record.getFields().stream().map(field -> {
95+
final var fieldName = toUserIdentifier ? DataTypeUtils.toUserIdentifier(field.getFieldName()) : field.getFieldName();
96+
return DataType.StructType.Field.from(fieldName, toRelationalType(field.getFieldType(), toUserIdentifier), field.getFieldIndex());
97+
}).collect(Collectors.toList());
98+
final var name = record.getName() == null ? getUniqueName() : (toUserIdentifier ? DataTypeUtils.toUserIdentifier(record.getName()) : record.getName());
99+
return DataType.StructType.from(name, columns, record.isNullable());
80100
case ARRAY:
81101
final var asArray = (Type.Array) type;
82102
return DataType.ArrayType.from(toRelationalType(Assert.notNullUnchecked(asArray.getElementType())), asArray.isNullable());
83103
case ENUM:
84104
final var asEnum = (Type.Enum) type;
85105
final var enumValues = asEnum.getEnumValues().stream().map(v -> DataType.EnumType.EnumValue.of(v.getName(), v.getNumber())).collect(Collectors.toList());
86-
return DataType.EnumType.from(asEnum.getName() == null ? toProtoBufCompliantName(UUID.randomUUID().toString()) : asEnum.getName(), enumValues, asEnum.isNullable());
106+
return DataType.EnumType.from(asEnum.getName() == null ? getUniqueName() : asEnum.getName(), enumValues, asEnum.isNullable());
87107
default:
88108
Assert.failUnchecked(String.format(Locale.ROOT, "unexpected type %s", type));
89109
return null; // make compiler happy.
90110
}
91111
}
92112

93113
@Nonnull
94-
private static String toProtoBufCompliantName(@Nonnull final String input) {
95-
Assert.thatUnchecked(input.length() > 0);
96-
final var modified = input.replace("-", "_");
97-
final char c = input.charAt(0);
114+
private static String getUniqueName() {
115+
final var uuid = UUID.randomUUID().toString();
116+
final var modified = uuid.replace("-", "_");
117+
final char c = uuid.charAt(0);
98118
if (c == '_' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')) {
99119
return modified;
100120
}
101121
return "id" + modified;
102122
}
103123

124+
public static String toProtoBufCompliantName(String userIdentifier) {
125+
return userIdentifier.replace("__", DOUBLE_UNDERSCORE_ESCAPE).replace("$", DOLLAR_ESCAPE).replace(".", DOT_ESCAPE);
126+
}
127+
128+
public static String toUserIdentifier(String protoIdentifier) {
129+
return protoIdentifier.replace(DOT_ESCAPE, ".").replace(DOLLAR_ESCAPE, "$").replace(DOUBLE_UNDERSCORE_ESCAPE, "__");
130+
}
131+
104132
/**
105133
* Converts a given Relational {@link DataType} into a corresponding Record Layer {@link Type}.
106134
*

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerInvokedRoutine.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.apple.foundationdb.record.query.plan.cascades.RawSqlFunction;
2424
import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
2525
import com.apple.foundationdb.relational.api.metadata.InvokedRoutine;
26+
import com.apple.foundationdb.relational.recordlayer.util.MemoizedFunction;
2627
import com.apple.foundationdb.relational.util.Assert;
2728

2829
import javax.annotation.Nonnull;
@@ -43,7 +44,7 @@ public class RecordLayerInvokedRoutine implements InvokedRoutine {
4344
private final boolean isTemporary;
4445

4546
@Nonnull
46-
private final Function<Boolean, UserDefinedFunction> userDefinedFunctionSupplier;
47+
private final Function<Boolean, UserDefinedFunction> userDefinedFunctionProvider;
4748

4849
@Nonnull
4950
private final UserDefinedFunction serializableFunction;
@@ -52,14 +53,14 @@ public RecordLayerInvokedRoutine(@Nonnull final String description,
5253
@Nonnull final String normalizedDescription,
5354
@Nonnull final String name,
5455
boolean isTemporary,
55-
@Nonnull final Function<Boolean, UserDefinedFunction> userDefinedFunctionSupplier,
56+
@Nonnull final Function<Boolean, UserDefinedFunction> userDefinedFunctionProvider,
5657
@Nonnull final UserDefinedFunction serializableFunction) {
5758
this.description = description;
5859
this.normalizedDescription = normalizedDescription;
5960
this.name = name;
6061
this.isTemporary = isTemporary;
6162
// TODO this used to be memoized
62-
this.userDefinedFunctionSupplier = userDefinedFunctionSupplier;
63+
this.userDefinedFunctionProvider = MemoizedFunction.memoize(userDefinedFunctionProvider::apply);
6364
this.serializableFunction = serializableFunction;
6465
}
6566

@@ -76,8 +77,8 @@ public String getNormalizedDescription() {
7677
}
7778

7879
@Nonnull
79-
public Function<Boolean, UserDefinedFunction> getUserDefinedFunctionSupplier() {
80-
return userDefinedFunctionSupplier;
80+
public Function<Boolean, UserDefinedFunction> getUserDefinedFunctionProvider() {
81+
return userDefinedFunctionProvider;
8182
}
8283

8384
@Nonnull
@@ -135,15 +136,15 @@ public Builder toBuilder() {
135136
.setDescription(getDescription())
136137
.setNormalizedDescription(getNormalizedDescription())
137138
.setTemporary(isTemporary())
138-
.withUserDefinedRoutine(getUserDefinedFunctionSupplier())
139+
.withUserDefinedRoutine(getUserDefinedFunctionProvider())
139140
.withSerializableFunction(asSerializableFunction());
140141
}
141142

142143
public static final class Builder {
143144
private String description;
144145
private String normalizedDescription;
145146
private String name;
146-
private Function<Boolean, UserDefinedFunction> userDefinedFunctionSupplier;
147+
private Function<Boolean, UserDefinedFunction> userDefinedFunctionProvider;
147148
private UserDefinedFunction serializableFunction;
148149
private boolean isTemporary;
149150

@@ -169,8 +170,8 @@ public Builder setName(@Nonnull final String name) {
169170
}
170171

171172
@Nonnull
172-
public Builder withUserDefinedRoutine(@Nonnull final Function<Boolean, UserDefinedFunction> userDefinedFunctionSupplier) {
173-
this.userDefinedFunctionSupplier = userDefinedFunctionSupplier;
173+
public Builder withUserDefinedRoutine(@Nonnull final Function<Boolean, UserDefinedFunction> userDefinedFunctionProvider) {
174+
this.userDefinedFunctionProvider = userDefinedFunctionProvider;
174175
return this;
175176
}
176177

@@ -190,10 +191,10 @@ public Builder setTemporary(boolean isTemporary) {
190191
public RecordLayerInvokedRoutine build() {
191192
Assert.notNullUnchecked(name);
192193
Assert.notNullUnchecked(description);
193-
Assert.notNullUnchecked(userDefinedFunctionSupplier);
194+
Assert.notNullUnchecked(userDefinedFunctionProvider);
194195
Assert.notNullUnchecked(serializableFunction);
195196
return new RecordLayerInvokedRoutine(description, normalizedDescription, name, isTemporary,
196-
userDefinedFunctionSupplier, serializableFunction);
197+
userDefinedFunctionProvider, serializableFunction);
197198
}
198199
}
199200
}

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ public static NormalizationResult normalizeAst(@Nonnull final SchemaTemplate sch
629629
// immediate materialization of temporary function, this is required to collect any auxiliary literals discovered
630630
// during plan generation of the temporary function. The literals and combined with query literals and provided
631631
// for the execution of a (cached) physical plan.
632-
final var compiledFunction = (CompiledSqlFunction)recordLayerRoutine.getUserDefinedFunctionSupplier().apply(caseSensitive);
632+
final var compiledFunction = (CompiledSqlFunction)recordLayerRoutine.getUserDefinedFunctionProvider().apply(caseSensitive);
633633
astNormalizer.queryHasherContextBuilder.getLiteralsBuilder().importLiterals(compiledFunction.getAuxiliaryLiterals());
634634
}
635635
return new NormalizationResult(

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,15 @@
2222

2323
import com.apple.foundationdb.annotation.API;
2424

25+
import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils;
2526
import com.google.common.collect.ImmutableList;
2627

2728
import javax.annotation.Nonnull;
2829
import java.util.Collection;
2930
import java.util.List;
3031
import java.util.Objects;
3132
import java.util.function.Function;
33+
import java.util.stream.Collectors;
3234

3335
@API(API.Status.EXPERIMENTAL)
3436
public class Identifier {
@@ -142,6 +144,13 @@ public boolean qualifiedWith(@Nonnull Identifier identifier) {
142144
return true;
143145
}
144146

147+
@Nonnull
148+
public static Identifier toProtobufCompliant(@Nonnull final Identifier identifier) {
149+
final var qualifier = identifier.getQualifier().stream().map(DataTypeUtils::toProtoBufCompliantName).collect(Collectors.toList());
150+
final var name = DataTypeUtils.toProtoBufCompliantName(identifier.getName());
151+
return Identifier.of(name, qualifier);
152+
}
153+
145154
@Override
146155
public boolean equals(Object obj) {
147156
if (obj == this) {

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PseudoColumn.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,13 @@ public static Optional<Expression> mapToExpressionMaybe(@Nonnull LogicalOperator
7373
}
7474
return Optional.empty();
7575
}
76+
77+
public static boolean isPseudoColumn(@Nonnull String name) {
78+
for (PseudoColumn pseudo : PseudoColumn.values()) {
79+
if (name.equals(pseudo.getColumnName())) {
80+
return true;
81+
}
82+
}
83+
return false;
84+
}
7685
}

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -404,7 +404,7 @@ private RelationalResultSet executePhysicalPlan(@Nonnull final RecordLayerSchema
404404
parsedContinuation.getExecutionState(),
405405
executeProperties));
406406
final var currentPlanHashMode = OptionsUtils.getCurrentPlanHashMode(options);
407-
final var dataType = (DataType.StructType) DataTypeUtils.toRelationalType(type);
407+
final var dataType = (DataType.StructType) DataTypeUtils.toRelationalType(type, true);
408408
return executionContext.metricCollector.clock(RelationalMetric.RelationalEvent.CREATE_RESULT_SET_ITERATOR, () -> {
409409
final ResumableIterator<Row> iterator = RecordLayerIterator.create(cursor, messageFDBQueriedRecord -> new MessageTuple(messageFDBQueriedRecord.getMessage()));
410410
return new RecordLayerResultSet(RelationalStructMetaData.of(dataType), iterator, connection,

‎fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -970,12 +970,13 @@ public LogicalOperator resolveTableFunction(@Nonnull final Identifier functionNa
970970
: tableFunction.encapsulate(valueArgs);
971971
if (resultingValue instanceof StreamingValue) {
972972
final var tableFunctionExpression = new TableFunctionExpression(Assert.castUnchecked(resultingValue, StreamingValue.class));
973-
final var resultingQuantifier = Quantifier.forEach(Reference.initialOf(tableFunctionExpression));
973+
final var reference = Reference.initialOf(tableFunctionExpression);
974+
final var resultingQuantifier = Quantifier.forEach(reference);
974975
final var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier));
975976
return LogicalOperator.newNamedOperator(functionName, output, resultingQuantifier);
976977
}
977-
final var relationalExpression = Assert.castUnchecked(resultingValue, RelationalExpression.class);
978-
final var topQun = Quantifier.forEach(Reference.initialOf(relationalExpression));
978+
final var tableExpression = Assert.castUnchecked(resultingValue, RelationalExpression.class);
979+
final var topQun = Quantifier.forEach(Reference.initialOf(tableExpression));
979980
return LogicalOperator.newNamedOperator(functionName, Expressions.fromQuantifier(topQun), topQun);
980981
}
981982

0 commit comments

Comments
 (0)