From b480e7df3439b14277f743db1781786b49463773 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 18 Mar 2026 16:21:26 -0400 Subject: [PATCH 1/2] impl(o11y): introduce resend_count logic --- .../gax/tracing/ObservabilityAttributes.java | 6 + .../api/gax/tracing/ObservabilityUtils.java | 2 + .../google/api/gax/tracing/SpanTracer.java | 35 +++++ .../api/gax/tracing/SpanTracerTest.java | 86 +++++++++++ .../showcase/v1beta1/it/ITOtelTracing.java | 143 ++++++++++++++++++ 5 files changed, 272 insertions(+) diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java index 6355dc0b9d..3e6487f2da 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java @@ -81,4 +81,10 @@ public class ObservabilityAttributes { /** The url template of the request (e.g. /v1/{name}:access). */ public static final String URL_TEMPLATE_ATTRIBUTE = "url.template"; + + /** The resend count of the request. Only used in HTTP transport. */ + public static final String HTTP_RESEND_COUNT_ATTRIBUTE = "http.request.resend_count"; + + /** The resend count of the request. Only used in gRPC transport. */ + public static final String GRPC_RESEND_COUNT_ATTRIBUTE = "gcp.grpc.resend_count"; } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java index f2a787fc95..2487964370 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java @@ -65,6 +65,8 @@ static Attributes toOtelAttributes(Map attributes) { (k, v) -> { if (v instanceof String) { attributesBuilder.put(k, (String) v); + } else if (v instanceof Long) { + attributesBuilder.put(k, (Long) v); } else if (v instanceof Integer) { attributesBuilder.put(k, (long) (Integer) v); } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java index c5c28aebe0..cee3f236ee 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java @@ -52,6 +52,7 @@ public class SpanTracer implements ApiTracer { private final String attemptSpanName; private final ApiTracerContext apiTracerContext; private TraceManager.Span attemptHandle; + private long resendCount; /** * Creates a new instance of {@code SpanTracer}. @@ -65,6 +66,7 @@ public SpanTracer( this.attemptSpanName = attemptSpanName; this.apiTracerContext = apiTracerContext; this.attemptAttributes = new HashMap<>(); + this.resendCount = 0; buildAttributes(); } @@ -76,8 +78,21 @@ private void buildAttributes() { @Override public void attemptStarted(Object request, int attemptNumber) { Map attemptAttributes = new HashMap<>(this.attemptAttributes); + + if (this.resendCount > 0) { + ApiTracerContext.Transport transport = apiTracerContext.transport(); + if (transport == ApiTracerContext.Transport.GRPC) { + attemptAttributes.put( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, this.resendCount); + } else if (transport == ApiTracerContext.Transport.HTTP) { + attemptAttributes.put( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, this.resendCount); + } + } + // Start the specific attempt span with the operation span as parent this.attemptHandle = traceManager.createSpan(attemptSpanName, attemptAttributes); + this.resendCount++; } @Override @@ -85,6 +100,26 @@ public void attemptSucceeded() { endAttempt(); } + @Override + public void attemptCancelled() { + endAttempt(); + } + + @Override + public void attemptFailedDuration(Throwable error, java.time.Duration delay) { + endAttempt(); + } + + @Override + public void attemptFailedRetriesExhausted(Throwable error) { + endAttempt(); + } + + @Override + public void attemptPermanentFailure(Throwable error) { + endAttempt(); + } + private void endAttempt() { if (attemptHandle != null) { attemptHandle.end(); diff --git a/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java b/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java index b5e6100fe6..8e35ba1f35 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java @@ -76,4 +76,90 @@ void testAttemptStarted_includesLanguageAttribute() { assertThat(attributesCaptor.getValue()) .containsEntry(SpanTracer.LANGUAGE_ATTRIBUTE, SpanTracer.DEFAULT_LANGUAGE); } + + @Test + void testAttemptStarted_retryAttributes_grpc() { + ApiTracerContext grpcContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.GRPC) + .build(); + SpanTracer grpcTracer = new SpanTracer(recorder, grpcContext, ATTEMPT_SPAN_NAME); + + // First attempt, no retry attribute + grpcTracer.attemptStarted(new Object(), 0); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Map.class); + verify(recorder).createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + assertThat(attributesCaptor.getValue()) + .doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE); + assertThat(attributesCaptor.getValue()) + .doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE); + + // First retry + grpcTracer.attemptStarted(new Object(), 0); + verify(recorder, org.mockito.Mockito.times(2)) + .createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + Map capturedAttributes = (Map) attributesCaptor.getValue(); + assertThat(capturedAttributes) + .containsEntry(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, 1L); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE); + + // N-th retry + grpcTracer.attemptStarted(new Object(), 0); + grpcTracer.attemptStarted(new Object(), 0); + grpcTracer.attemptStarted(new Object(), 0); + grpcTracer.attemptStarted(new Object(), 0); + verify(recorder, org.mockito.Mockito.times(6)) + .createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + capturedAttributes = (Map) attributesCaptor.getValue(); + assertThat(capturedAttributes) + .containsEntry(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, 5L); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE); + } + + @Test + void testAttemptStarted_retryAttributes_http() { + ApiTracerContext httpContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.HTTP) + .build(); + SpanTracer httpTracer = new SpanTracer(recorder, httpContext, ATTEMPT_SPAN_NAME); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Map.class); + + // First attempt, no retry attribute + httpTracer.attemptStarted(new Object(), 0); + verify(recorder, org.mockito.Mockito.times(1)) + .createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + Map capturedAttributes = (Map) attributesCaptor.getValue(); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE); + + // First retry + httpTracer.attemptStarted(new Object(), 0); + verify(recorder, org.mockito.Mockito.times(2)) + .createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + capturedAttributes = (Map) attributesCaptor.getValue(); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE); + assertThat(capturedAttributes) + .containsEntry(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, 1L); + + // N-th retry + httpTracer.attemptStarted(new Object(), 0); + httpTracer.attemptStarted(new Object(), 0); + httpTracer.attemptStarted(new Object(), 0); + httpTracer.attemptStarted(new Object(), 0); + verify(recorder, org.mockito.Mockito.times(6)) + .createSpan(eq(ATTEMPT_SPAN_NAME), attributesCaptor.capture()); + capturedAttributes = (Map) attributesCaptor.getValue(); + assertThat(capturedAttributes) + .doesNotContainKey(ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE); + assertThat(capturedAttributes) + .containsEntry(ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, 5L); + } } diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java index d8ca71e848..7dbcf42e50 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java @@ -31,14 +31,24 @@ package com.google.showcase.v1beta1.it; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.StatusCode; +import com.google.api.gax.rpc.UnavailableException; import com.google.api.gax.tracing.ObservabilityAttributes; import com.google.api.gax.tracing.OpenTelemetryTraceManager; import com.google.api.gax.tracing.SpanTracer; import com.google.api.gax.tracing.SpanTracerFactory; +import com.google.rpc.Status; import com.google.showcase.v1beta1.EchoClient; import com.google.showcase.v1beta1.EchoRequest; +import com.google.showcase.v1beta1.EchoSettings; import com.google.showcase.v1beta1.it.util.TestClientInitializer; +import com.google.showcase.v1beta1.stub.EchoStub; +import com.google.showcase.v1beta1.stub.EchoStubSettings; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; @@ -48,6 +58,7 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -196,4 +207,136 @@ void testTracing_successfulEcho_httpjson() throws Exception { .isEqualTo("v1beta1/echo:echo"); } } + + @Test + void testTracing_retry_grpc() throws Exception { + final int attempts = 5; + final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE; + // A custom EchoClient is used in this test because retries have jitter, and we cannot + // predict the number of attempts that are scheduled for an RPC invocation otherwise. + // The custom retrySettings limit to a set number of attempts before the call gives up. + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L)) + .setMaxAttempts(attempts) + .build(); + + EchoStubSettings.Builder grpcEchoSettingsBuilder = EchoStubSettings.newBuilder(); + grpcEchoSettingsBuilder + .echoSettings() + .setRetrySettings(retrySettings) + .setRetryableCodes(statusCode); + EchoSettings grpcEchoSettings = EchoSettings.create(grpcEchoSettingsBuilder.build()); + grpcEchoSettings = + grpcEchoSettings.toBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider(EchoSettings.defaultGrpcTransportProviderBuilder().build()) + .setEndpoint("localhost:7469") + .build(); + + SpanTracerFactory tracingFactory = + new SpanTracerFactory(new OpenTelemetryTraceManager(openTelemetrySdk)); + + EchoStubSettings echoStubSettings = + (EchoStubSettings) + grpcEchoSettings.getStubSettings().toBuilder().setTracerFactory(tracingFactory).build(); + EchoStub stub = echoStubSettings.createStub(); + EchoClient grpcClient = EchoClient.create(stub); + + EchoRequest echoRequest = + EchoRequest.newBuilder() + .setError(Status.newBuilder().setCode(statusCode.ordinal()).build()) + .build(); + + assertThrows(UnavailableException.class, () -> grpcClient.echo(echoRequest)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry + + // This single span represents the successful retry, which has resend_count=1 + for (int resendCount = 1; resendCount < attempts; resendCount++) { + Optional found = + spans.stream() + .filter( + span -> + span.getAttributes() + .asMap() + .getOrDefault( + AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE), + -1L) + .equals(1L)) + .findFirst(); + assertThat(found).isPresent(); + } + } + + @Test + void testTracing_retry_httpjson() throws Exception { + final int attempts = 5; + final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE; + // A custom EchoClient is used in this test because retries have jitter, and we cannot + // predict the number of attempts that are scheduled for an RPC invocation otherwise. + // The custom retrySettings limit to a set number of attempts before the call gives up. + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L)) + .setMaxAttempts(attempts) + .build(); + + EchoStubSettings.Builder httpJsonEchoSettingsBuilder = EchoStubSettings.newHttpJsonBuilder(); + httpJsonEchoSettingsBuilder + .echoSettings() + .setRetrySettings(retrySettings) + .setRetryableCodes(statusCode); + EchoSettings httpJsonEchoSettings = EchoSettings.create(httpJsonEchoSettingsBuilder.build()); + httpJsonEchoSettings = + httpJsonEchoSettings.toBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultHttpJsonTransportProviderBuilder() + .setHttpTransport( + new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .setEndpoint("http://localhost:7469") + .build()) + .build(); + + SpanTracerFactory tracingFactory = + new SpanTracerFactory(new OpenTelemetryTraceManager(openTelemetrySdk)); + + EchoStubSettings echoStubSettings = + (EchoStubSettings) + httpJsonEchoSettings.getStubSettings().toBuilder() + .setTracerFactory(tracingFactory) + .build(); + EchoStub stub = echoStubSettings.createStub(); + EchoClient httpClient = EchoClient.create(stub); + + EchoRequest echoRequest = + EchoRequest.newBuilder() + .setError(Status.newBuilder().setCode(statusCode.ordinal()).build()) + .build(); + + assertThrows(UnavailableException.class, () -> httpClient.echo(echoRequest)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry + + // This single span represents the successful retry, which has resend_count=1 + for (int resendCount = 1; resendCount < attempts; resendCount++) { + Optional found = + spans.stream() + .filter( + span -> + span.getAttributes() + .asMap() + .getOrDefault( + AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE), + -1L) + .equals(1L)) + .findFirst(); + assertThat(found).isPresent(); + } + } } From 9b2734c098f36cdf44ed669481709b2791211dbc Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 18 Mar 2026 22:13:32 -0400 Subject: [PATCH 2/2] test: simplify verification logic --- .../showcase/v1beta1/it/ITOtelTracing.java | 73 +++++++++++-------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java index 7dbcf42e50..2b86433303 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java @@ -58,7 +58,6 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.util.List; -import java.util.Optional; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -254,21 +253,27 @@ void testTracing_retry_grpc() throws Exception { assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry // This single span represents the successful retry, which has resend_count=1 - for (int resendCount = 1; resendCount < attempts; resendCount++) { - Optional found = - spans.stream() - .filter( - span -> - span.getAttributes() - .asMap() - .getOrDefault( - AttributeKey.longKey( - ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE), - -1L) - .equals(1L)) - .findFirst(); - assertThat(found).isPresent(); - } + // The first attempt has no resend_count. The subsequent retries will have a resend_count, + // starting from 1. + List resendCounts = + spans.stream() + .map( + span -> + (Long) + span.getAttributes() + .asMap() + .get( + AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE))) + .filter(java.util.Objects::nonNull) + .sorted() + .collect(java.util.stream.Collectors.toList()); + + List expectedCounts = + java.util.stream.LongStream.range(1, attempts) + .boxed() + .collect(java.util.stream.Collectors.toList()); + assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder(); } @Test @@ -323,20 +328,26 @@ void testTracing_retry_httpjson() throws Exception { assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry // This single span represents the successful retry, which has resend_count=1 - for (int resendCount = 1; resendCount < attempts; resendCount++) { - Optional found = - spans.stream() - .filter( - span -> - span.getAttributes() - .asMap() - .getOrDefault( - AttributeKey.longKey( - ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE), - -1L) - .equals(1L)) - .findFirst(); - assertThat(found).isPresent(); - } + // The first attempt has no resend_count. The subsequent retries will have a resend_count, + // starting from 1. + List resendCounts = + spans.stream() + .map( + span -> + (Long) + span.getAttributes() + .asMap() + .get( + AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE))) + .filter(java.util.Objects::nonNull) + .sorted() + .collect(java.util.stream.Collectors.toList()); + + List expectedCounts = + java.util.stream.LongStream.range(1, attempts) + .boxed() + .collect(java.util.stream.Collectors.toList()); + assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder(); } }