Skip to content

Commit bb82f9e

Browse files
rahul2393rahul2393
authored
feat: make grpc-gcp default enabled (#4239)
This PR enables the gRPC-GCP channel pool extension by default for Cloud Spanner Java client. **What's Changing for Customers** **Before this change** - gRPC-GCP extension was disabled by default - Default number of channels: 4 - Channel pooling was handled by GAX **After this change** - gRPC-GCP extension is enabled by default - Default number of channels: 8 - Channel pooling is handled by gRPC-GCP extension **Benefits of gRPC-GCP** - **Improved resilience:** When a network connection fails on a particular channel, operations can be automatically retried on a different gRPC channel - **Better channel management:** gRPC-GCP provides more sophisticated channel affinity and load balancing **How to Disable gRPC-GCP (Switch Back to GAX Channel Pool)** If you need to disable gRPC-GCP and use the previous GAX channel pooling behavior, use the `disableGrpcGcpExtension()` method: ``` SpannerOptions options = SpannerOptions.newBuilder() .setProjectId("my-project") .disableGrpcGcpExtension() .build(); ``` When disabled, the default number of channels reverts to 4 (the previous default). **When You Might Want to Disable gRPC-GCP** - **Maintaining previous behavior:** If you want to keep the exact same behavior as before this change (GAX channel pool with 4 default channels). - **Troubleshooting**: If you experience any unexpected behavior, disabling gRPC-GCP can help isolate whether the issue is related to the channel pooling mechanism.
1 parent afd7d6b commit bb82f9e

File tree

11 files changed

+211
-145
lines changed

11 files changed

+211
-145
lines changed

‎google-cloud-spanner-executor/pom.xml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@
5959
</exclusion>
6060
</exclusions>
6161
</dependency>
62+
<dependency>
63+
<groupId>com.google.cloud</groupId>
64+
<artifactId>grpc-gcp</artifactId>
65+
<version>${grpc.gcp.version}</version>
66+
</dependency>
6267
<dependency>
6368
<groupId>io.opentelemetry.semconv</groupId>
6469
<artifactId>opentelemetry-semconv</artifactId>
@@ -296,7 +301,7 @@
296301
<groupId>org.apache.maven.plugins</groupId>
297302
<artifactId>maven-dependency-plugin</artifactId>
298303
<configuration>
299-
<ignoredDependencies> com.google.api:gax,org.apache.maven.surefire:surefire-junit4,io.opentelemetry.semconv:opentelemetry-semconv,com.google.cloud.opentelemetry:shared-resourcemapping </ignoredDependencies>
304+
<ignoredDependencies> com.google.api:gax,org.apache.maven.surefire:surefire-junit4,io.opentelemetry.semconv:opentelemetry-semconv,com.google.cloud.opentelemetry:shared-resourcemapping,com.google.cloud:grpc-gcp </ignoredDependencies>
300305
</configuration>
301306
</plugin>
302307
</plugins>

‎google-cloud-spanner/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@
166166
<dependency>
167167
<groupId>com.google.cloud</groupId>
168168
<artifactId>grpc-gcp</artifactId>
169+
<version>${grpc.gcp.version}</version>
169170
</dependency>
170171
<dependency>
171172
<groupId>io.grpc</groupId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,7 +1025,7 @@ public static class Builder
10251025
private DatabaseAdminStubSettings.Builder databaseAdminStubSettingsBuilder =
10261026
DatabaseAdminStubSettings.newBuilder();
10271027
private Duration partitionedDmlTimeout = Duration.ofHours(2L);
1028-
private boolean grpcGcpExtensionEnabled = false;
1028+
private boolean grpcGcpExtensionEnabled = true;
10291029
private GcpManagedChannelOptions grpcGcpOptions;
10301030
private RetrySettings retryAdministrativeRequestsSettings =
10311031
DEFAULT_ADMIN_REQUESTS_LIMIT_EXCEEDED_RETRY_SETTINGS;
@@ -1557,28 +1557,22 @@ public Builder setExperimentalHost(String host) {
15571557
return this;
15581558
}
15591559

1560-
/**
1561-
* Enables gRPC-GCP extension with the default settings. Do not set
1562-
* GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS to true in combination with this option, as
1563-
* Multiplexed sessions are not supported for gRPC-GCP.
1564-
*/
1560+
/** Enables gRPC-GCP extension with the default settings. This option is enabled by default. */
15651561
public Builder enableGrpcGcpExtension() {
15661562
return this.enableGrpcGcpExtension(null);
15671563
}
15681564

15691565
/**
15701566
* Enables gRPC-GCP extension and uses provided options for configuration. The metric registry
1571-
* and default Spanner metric labels will be added automatically. Do not set
1572-
* GOOGLE_CLOUD_SPANNER_MULTIPLEXED_SESSIONS to true in combination with this option, as
1573-
* Multiplexed sessions are not supported for gRPC-GCP.
1567+
* and default Spanner metric labels will be added automatically.
15741568
*/
15751569
public Builder enableGrpcGcpExtension(GcpManagedChannelOptions options) {
15761570
this.grpcGcpExtensionEnabled = true;
15771571
this.grpcGcpOptions = options;
15781572
return this;
15791573
}
15801574

1581-
/** Disables gRPC-GCP extension. */
1575+
/** Disables gRPC-GCP extension and uses GAX channel pool instead. */
15821576
public Builder disableGrpcGcpExtension() {
15831577
this.grpcGcpExtensionEnabled = false;
15841578
return this;

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,6 @@
188188
import io.grpc.Context;
189189
import io.grpc.ManagedChannelBuilder;
190190
import io.grpc.MethodDescriptor;
191-
import io.opencensus.metrics.Metrics;
192191
import java.io.IOException;
193192
import java.io.UnsupportedEncodingException;
194193
import java.net.URLDecoder;
@@ -570,17 +569,14 @@ private static String parseGrpcGcpApiConfig() {
570569
}
571570
}
572571

573-
// Enhance metric options for gRPC-GCP extension. Adds metric registry if not specified.
572+
// Enhance metric options for gRPC-GCP extension.
574573
private static GcpManagedChannelOptions grpcGcpOptionsWithMetrics(SpannerOptions options) {
575574
GcpManagedChannelOptions grpcGcpOptions =
576575
MoreObjects.firstNonNull(options.getGrpcGcpOptions(), new GcpManagedChannelOptions());
577576
GcpMetricsOptions metricsOptions =
578577
MoreObjects.firstNonNull(
579578
grpcGcpOptions.getMetricsOptions(), GcpMetricsOptions.newBuilder().build());
580579
GcpMetricsOptions.Builder metricsOptionsBuilder = GcpMetricsOptions.newBuilder(metricsOptions);
581-
if (metricsOptions.getMetricRegistry() == null) {
582-
metricsOptionsBuilder.withMetricRegistry(Metrics.getMetricRegistry());
583-
}
584580
// TODO: Add default labels with values: client_id, database, instance_id.
585581
if (metricsOptions.getNamePrefix().equals("")) {
586582
metricsOptionsBuilder.withNamePrefix("cloud.google.com/java/spanner/gcp-channel-pool/");

‎google-cloud-spanner/src/main/resources/META-INF/native-image/com.google.cloud.spanner/grpc-gcp-reflect-config.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[
2+
{
3+
"name": "com.google.cloud.grpc.proto.ApiConfig",
4+
"allDeclaredFields": true,
5+
"allDeclaredMethods": true,
6+
"allDeclaredConstructors": true
7+
},
8+
{
9+
"name": "com.google.cloud.grpc.proto.ApiConfig$Builder",
10+
"allDeclaredFields": true,
11+
"allDeclaredMethods": true,
12+
"allDeclaredConstructors": true
13+
},
14+
{
15+
"name": "com.google.cloud.grpc.proto.ChannelPoolConfig",
16+
"allDeclaredFields": true,
17+
"allDeclaredMethods": true,
18+
"allDeclaredConstructors": true
19+
},
20+
{
21+
"name": "com.google.cloud.grpc.proto.ChannelPoolConfig$Builder",
22+
"allDeclaredFields": true,
23+
"allDeclaredMethods": true,
24+
"allDeclaredConstructors": true
25+
},
26+
{
27+
"name": "com.google.cloud.grpc.proto.MethodConfig",
28+
"allDeclaredFields": true,
29+
"allDeclaredMethods": true,
30+
"allDeclaredConstructors": true
31+
},
32+
{
33+
"name": "com.google.cloud.grpc.proto.MethodConfig$Builder",
34+
"allDeclaredFields": true,
35+
"allDeclaredMethods": true,
36+
"allDeclaredConstructors": true
37+
},
38+
{
39+
"name": "com.google.cloud.grpc.proto.AffinityConfig",
40+
"allDeclaredFields": true,
41+
"allDeclaredMethods": true,
42+
"allDeclaredConstructors": true
43+
},
44+
{
45+
"name": "com.google.cloud.grpc.proto.AffinityConfig$Builder",
46+
"allDeclaredFields": true,
47+
"allDeclaredMethods": true,
48+
"allDeclaredConstructors": true
49+
},
50+
{
51+
"name": "com.google.cloud.grpc.proto.AffinityConfig$Command",
52+
"allDeclaredFields": true,
53+
"allDeclaredMethods": true,
54+
"allDeclaredConstructors": true
55+
}
56+
]

‎google-cloud-spanner/src/main/resources/META-INF/native-image/native-image.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ Args = --initialize-at-build-time=com.google.cloud.spanner.IntegrationTestEnv,\
22
org.junit.experimental.categories.CategoryValidator,\
33
org.junit.validator.AnnotationValidator,\
44
java.lang.annotation.Annotation \
5+
-H:ReflectionConfigurationResources=${.}/com.google.cloud.spanner/grpc-gcp-reflect-config.json \
56
--features=com.google.cloud.spanner.nativeimage.SpannerFeature

‎google-cloud-spanner/src/test/java/com/google/cloud/spanner/ChannelUsageTest.java‎

Lines changed: 47 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,9 @@
1717
package com.google.cloud.spanner;
1818

1919
import static com.google.cloud.spanner.DisableDefaultMtlsProvider.disableDefaultMtlsProvider;
20-
import static io.grpc.Grpc.TRANSPORT_ATTR_REMOTE_ADDR;
20+
import static java.util.stream.Collectors.toSet;
2121
import static org.junit.Assert.assertEquals;
2222
import static org.junit.Assert.assertTrue;
23-
import static org.junit.Assume.assumeFalse;
2423

2524
import com.google.cloud.NoCredentials;
2625
import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult;
@@ -32,7 +31,6 @@
3231
import com.google.spanner.v1.StructType;
3332
import com.google.spanner.v1.StructType.Field;
3433
import com.google.spanner.v1.TypeCode;
35-
import io.grpc.Attributes;
3634
import io.grpc.Context;
3735
import io.grpc.Contexts;
3836
import io.grpc.Metadata;
@@ -70,13 +68,9 @@ public class ChannelUsageTest {
7068
@Parameter(0)
7169
public int numChannels;
7270

73-
@Parameter(1)
74-
public boolean enableGcpPool;
75-
76-
@Parameters(name = "num channels = {0}, enable GCP pool = {1}")
71+
@Parameters(name = "num channels = {0}")
7772
public static Collection<Object[]> data() {
78-
return Arrays.asList(
79-
new Object[][] {{1, true}, {1, false}, {2, true}, {2, false}, {4, true}, {4, false}});
73+
return Arrays.asList(new Object[][] {{1}, {2}, {4}});
8074
}
8175

8276
private static final Statement SELECT1 = Statement.of("SELECT 1 AS COL1");
@@ -106,9 +100,9 @@ public static Collection<Object[]> data() {
106100
private static MockSpannerServiceImpl mockSpanner;
107101
private static Server server;
108102
private static InetSocketAddress address;
109-
private static final Set<InetSocketAddress> batchCreateSessionLocalIps =
110-
ConcurrentHashMap.newKeySet();
111-
private static final Set<InetSocketAddress> executeSqlLocalIps = ConcurrentHashMap.newKeySet();
103+
// Track channel hints (from X-Goog-Spanner-Request-Id header) per RPC method
104+
private static final Set<Long> batchCreateSessionChannelHints = ConcurrentHashMap.newKeySet();
105+
private static final Set<Long> executeSqlChannelHints = ConcurrentHashMap.newKeySet();
112106

113107
private static Level originalLogLevel;
114108

@@ -123,8 +117,8 @@ public static void startServer() throws Exception {
123117
server =
124118
NettyServerBuilder.forAddress(address)
125119
.addService(mockSpanner)
126-
// Add a server interceptor to register the remote addresses that we are seeing. This
127-
// indicates how many channels are used client side to communicate with the server.
120+
// Add a server interceptor to extract channel hints from X-Goog-Spanner-Request-Id
121+
// header. This verifies that the client uses all configured channels.
128122
.intercept(
129123
new ServerInterceptor() {
130124
@Override
@@ -138,22 +132,26 @@ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
138132
headers.get(
139133
Metadata.Key.of(
140134
"x-response-encoding", Metadata.ASCII_STRING_MARSHALLER)));
141-
Attributes attributes = call.getAttributes();
142-
@SuppressWarnings({"unchecked", "deprecation"})
143-
Attributes.Key<InetSocketAddress> key =
144-
(Attributes.Key<InetSocketAddress>)
145-
attributes.keys().stream()
146-
.filter(k -> k.equals(TRANSPORT_ATTR_REMOTE_ADDR))
147-
.findFirst()
148-
.orElse(null);
149-
if (key != null) {
150-
if (call.getMethodDescriptor()
151-
.equals(SpannerGrpc.getBatchCreateSessionsMethod())) {
152-
batchCreateSessionLocalIps.add(attributes.get(key));
153-
}
154-
if (call.getMethodDescriptor()
155-
.equals(SpannerGrpc.getExecuteStreamingSqlMethod())) {
156-
executeSqlLocalIps.add(attributes.get(key));
135+
// Extract channel hint from X-Goog-Spanner-Request-Id header
136+
String requestId = headers.get(XGoogSpannerRequestId.REQUEST_ID_HEADER_KEY);
137+
if (requestId != null) {
138+
// Format:
139+
// <version>.<randProcessId>.<nthClientId>.<nthChannelId>.<nthRequest>.<attempt>
140+
String[] parts = requestId.split("\\.");
141+
if (parts.length >= 4) {
142+
try {
143+
long channelHint = Long.parseLong(parts[3]);
144+
if (call.getMethodDescriptor()
145+
.equals(SpannerGrpc.getBatchCreateSessionsMethod())) {
146+
batchCreateSessionChannelHints.add(channelHint);
147+
}
148+
if (call.getMethodDescriptor()
149+
.equals(SpannerGrpc.getExecuteStreamingSqlMethod())) {
150+
executeSqlChannelHints.add(channelHint);
151+
}
152+
} catch (NumberFormatException e) {
153+
// Ignore parse errors
154+
}
157155
}
158156
}
159157
return Contexts.interceptCall(Context.current(), call, headers, next);
@@ -185,8 +183,8 @@ public static void resetLogging() {
185183
@After
186184
public void reset() {
187185
mockSpanner.reset();
188-
batchCreateSessionLocalIps.clear();
189-
executeSqlLocalIps.clear();
186+
batchCreateSessionChannelHints.clear();
187+
executeSqlChannelHints.clear();
190188
}
191189

192190
private SpannerOptions createSpannerOptions() {
@@ -208,34 +206,14 @@ private SpannerOptions createSpannerOptions() {
208206
.build())
209207
.setHost("http://" + endpoint)
210208
.setCredentials(NoCredentials.getInstance());
211-
if (enableGcpPool) {
212-
builder.enableGrpcGcpExtension();
213-
}
214209

215210
return builder.build();
216211
}
217212

218-
@Test
219-
public void testCreatesNumChannels() {
220-
try (Spanner spanner = createSpannerOptions().getService()) {
221-
assumeFalse(
222-
"GRPC-GCP is currently not supported with multiplexed sessions",
223-
isMultiplexedSessionsEnabled(spanner) && enableGcpPool);
224-
DatabaseClient client = spanner.getDatabaseClient(DatabaseId.of("p", "i", "d"));
225-
try (ResultSet resultSet = client.singleUse().executeQuery(SELECT1)) {
226-
while (resultSet.next()) {}
227-
}
228-
}
229-
assertEquals(numChannels, batchCreateSessionLocalIps.size());
230-
}
231-
232213
@Test
233214
public void testUsesAllChannels() throws InterruptedException {
234215
final int multiplier = 2;
235216
try (Spanner spanner = createSpannerOptions().getService()) {
236-
assumeFalse(
237-
"GRPC-GCP is currently not supported with multiplexed sessions",
238-
isMultiplexedSessionsEnabled(spanner));
239217
DatabaseClient client = spanner.getDatabaseClient(DatabaseId.of("p", "i", "d"));
240218
ListeningExecutorService executor =
241219
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numChannels * multiplier));
@@ -263,13 +241,23 @@ public void testUsesAllChannels() throws InterruptedException {
263241
executor.shutdown();
264242
assertTrue(executor.awaitTermination(Duration.ofSeconds(10L)));
265243
}
266-
assertEquals(numChannels, executeSqlLocalIps.size());
267-
}
268-
269-
private boolean isMultiplexedSessionsEnabled(Spanner spanner) {
270-
if (spanner.getOptions() == null || spanner.getOptions().getSessionPoolOptions() == null) {
271-
return false;
244+
// Bound the channel hints to numChannels (matching gRPC-GCP behavior) and verify
245+
// that channels are being distributed. The raw channel hints may be unbounded (based on
246+
// session index), but gRPC-GCP bounds them to the actual number of channels.
247+
Set<Long> boundedChannelHints =
248+
executeSqlChannelHints.stream().map(hint -> hint % numChannels).collect(toSet());
249+
// Verify that channel distribution is working:
250+
// - For numChannels=1, exactly 1 channel should be used
251+
// - For numChannels>1, multiple channels should be used (at least half)
252+
if (numChannels == 1) {
253+
assertEquals(1, boundedChannelHints.size());
254+
} else {
255+
assertTrue(
256+
"Expected at least "
257+
+ (numChannels / 2)
258+
+ " channels to be used, but got "
259+
+ boundedChannelHints.size(),
260+
boundedChannelHints.size() >= numChannels / 2);
272261
}
273-
return spanner.getOptions().getSessionPoolOptions().getUseMultiplexedSession();
274262
}
275263
}

0 commit comments

Comments
 (0)