otel: update the retry stats as per gRFC A96

AgraVator · AgraVator · commit a7a16ce0c75e · 2025-05-27T12:30:21.000+05:30
diff --git a/api/src/main/java/io/grpc/ClientStreamTracer.java b/api/src/main/java/io/grpc/ClientStreamTracer.java
@@ -132,12 +132,15 @@ public static final class StreamInfo {
     private final CallOptions callOptions;
     private final int previousAttempts;
     private final boolean isTransparentRetry;
+    private final boolean isHedging;
 
     StreamInfo(
-        CallOptions callOptions, int previousAttempts, boolean isTransparentRetry) {
+        CallOptions callOptions, int previousAttempts, boolean isTransparentRetry,
+        boolean isHedging) {
       this.callOptions = checkNotNull(callOptions, "callOptions");
       this.previousAttempts = previousAttempts;
       this.isTransparentRetry = isTransparentRetry;
+      this.isHedging = isHedging;
     }
 
     /**
@@ -165,6 +168,15 @@ public boolean isTransparentRetry() {
       return isTransparentRetry;
     }
 
+    /**
+     * Whether the stream is hedging.
+     *
+     * @since 1.74.0
+     */
+    public boolean isHedging() {
+      return isHedging;
+    }
+
     /**
      * Converts this StreamInfo into a new Builder.
      *
@@ -174,7 +186,9 @@ public Builder toBuilder() {
       return new Builder()
           .setCallOptions(callOptions)
           .setPreviousAttempts(previousAttempts)
-          .setIsTransparentRetry(isTransparentRetry);
+          .setIsTransparentRetry(isTransparentRetry)
+          .setIsHedging(isHedging);
+
     }
 
     /**
@@ -192,6 +206,7 @@ public String toString() {
           .add("callOptions", callOptions)
           .add("previousAttempts", previousAttempts)
           .add("isTransparentRetry", isTransparentRetry)
+          .add("isHedging", isHedging)
           .toString();
     }
 
@@ -204,6 +219,7 @@ public static final class Builder {
       private CallOptions callOptions = CallOptions.DEFAULT;
       private int previousAttempts;
       private boolean isTransparentRetry;
+      private boolean isHedging;
 
       Builder() {
       }
@@ -236,11 +252,21 @@ public Builder setIsTransparentRetry(boolean isTransparentRetry) {
         return this;
       }
 
+      /**
+       * Sets whether the stream is hedging.
+       *
+       * @since 1.74.0
+       */
+      public Builder setIsHedging(boolean isHedging) {
+        this.isHedging = isHedging;
+        return this;
+      }
+
       /**
        * Builds a new StreamInfo.
        */
       public StreamInfo build() {
-        return new StreamInfo(callOptions, previousAttempts, isTransparentRetry);
+        return new StreamInfo(callOptions, previousAttempts, isTransparentRetry, isHedging);
       }
     }
   }
diff --git a/core/src/main/java/io/grpc/internal/ClientCallImpl.java b/core/src/main/java/io/grpc/internal/ClientCallImpl.java
@@ -250,7 +250,8 @@ public void runInContext() {
       stream = clientStreamProvider.newStream(method, callOptions, headers, context);
     } else {
       ClientStreamTracer[] tracers =
-          GrpcUtil.getClientStreamTracers(callOptions, headers, 0, false);
+          GrpcUtil.getClientStreamTracers(callOptions, headers, 0,
+              false, false);
       String deadlineName = contextIsDeadlineSource ? "Context" : "CallOptions";
       Long nameResolutionDelay = callOptions.getOption(NAME_RESOLUTION_DELAYED);
       String description = String.format(
diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java
@@ -757,13 +757,15 @@ public ListenableFuture<SocketStats> getStats() {
 
   /** Gets stream tracers based on CallOptions. */
   public static ClientStreamTracer[] getClientStreamTracers(
-      CallOptions callOptions, Metadata headers, int previousAttempts, boolean isTransparentRetry) {
+      CallOptions callOptions, Metadata headers, int previousAttempts, boolean isTransparentRetry,
+      boolean isHedging) {
     List<ClientStreamTracer.Factory> factories = callOptions.getStreamTracerFactories();
     ClientStreamTracer[] tracers = new ClientStreamTracer[factories.size() + 1];
     StreamInfo streamInfo = StreamInfo.newBuilder()
         .setCallOptions(callOptions)
         .setPreviousAttempts(previousAttempts)
         .setIsTransparentRetry(isTransparentRetry)
+        .setIsHedging(isHedging)
         .build();
     for (int i = 0; i < factories.size(); i++) {
       tracers[i] = factories.get(i).newClientStreamTracer(streamInfo, headers);
diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
@@ -480,7 +480,8 @@ public ClientStream newStream(
       // the delayed transport or a real transport will go in-use and cancel the idle timer.
       if (!retryEnabled) {
         ClientStreamTracer[] tracers = GrpcUtil.getClientStreamTracers(
-            callOptions, headers, 0, /* isTransparentRetry= */ false);
+            callOptions, headers, 0, /* isTransparentRetry= */ false,
+            /* isHedging= */false);
         Context origContext = context.attach();
         try {
           return delayedTransport.newStream(method, headers, callOptions, tracers);
@@ -520,10 +521,10 @@ void postCommit() {
           @Override
           ClientStream newSubstream(
               Metadata newHeaders, ClientStreamTracer.Factory factory, int previousAttempts,
-              boolean isTransparentRetry) {
+              boolean isTransparentRetry, boolean isHedging) {
             CallOptions newOptions = callOptions.withStreamTracerFactory(factory);
             ClientStreamTracer[] tracers = GrpcUtil.getClientStreamTracers(
-                newOptions, newHeaders, previousAttempts, isTransparentRetry);
+                newOptions, newHeaders, previousAttempts, isTransparentRetry, isHedging);
             Context origContext = context.attach();
             try {
               return delayedTransport.newStream(method, newHeaders, newOptions, tracers);
diff --git a/core/src/main/java/io/grpc/internal/OobChannel.java b/core/src/main/java/io/grpc/internal/OobChannel.java
@@ -88,7 +88,8 @@ final class OobChannel extends ManagedChannel implements InternalInstrumented<Ch
     public ClientStream newStream(MethodDescriptor<?, ?> method,
         CallOptions callOptions, Metadata headers, Context context) {
       ClientStreamTracer[] tracers = GrpcUtil.getClientStreamTracers(
-          callOptions, headers, 0, /* isTransparentRetry= */ false);
+          callOptions, headers, 0, /* isTransparentRetry= */ false,
+          /* isHedging= */ false);
       Context origContext = context.attach();
       // delayed transport's newStream() always acquires a lock, but concurrent performance doesn't
       // matter here because OOB communication should be sparse, and it's not on application RPC's
diff --git a/core/src/main/java/io/grpc/internal/RetriableStream.java b/core/src/main/java/io/grpc/internal/RetriableStream.java
@@ -266,7 +266,8 @@ public ClientStreamTracer newClientStreamTracer(
 
     Metadata newHeaders = updateHeaders(headers, previousAttemptCount);
     // NOTICE: This set _must_ be done before stream.start() and it actually is.
-    sub.stream = newSubstream(newHeaders, tracerFactory, previousAttemptCount, isTransparentRetry);
+    sub.stream = newSubstream(newHeaders, tracerFactory, previousAttemptCount, isTransparentRetry,
+        isHedging);
     return sub;
   }
 
@@ -276,7 +277,7 @@ public ClientStreamTracer newClientStreamTracer(
    */
   abstract ClientStream newSubstream(
       Metadata headers, ClientStreamTracer.Factory tracerFactory, int previousAttempts,
-      boolean isTransparentRetry);
+      boolean isTransparentRetry, boolean isHedging);
 
   /** Adds grpc-previous-rpc-attempts in the headers of a retry/hedging RPC. */
   @VisibleForTesting
diff --git a/core/src/main/java/io/grpc/internal/SubchannelChannel.java b/core/src/main/java/io/grpc/internal/SubchannelChannel.java
@@ -59,7 +59,8 @@ public ClientStream newStream(MethodDescriptor<?, ?> method,
           transport = notReadyTransport;
         }
         ClientStreamTracer[] tracers = GrpcUtil.getClientStreamTracers(
-            callOptions, headers, 0, /* isTransparentRetry= */ false);
+            callOptions, headers, 0, /* isTransparentRetry= */ false,
+            /* isHedging= */ false);
         Context origContext = context.attach();
         try {
           return transport.newStream(method, headers, callOptions, tracers);
diff --git a/core/src/test/java/io/grpc/internal/RetriableStreamTest.java b/core/src/test/java/io/grpc/internal/RetriableStreamTest.java
@@ -186,7 +186,8 @@ ClientStream newSubstream(
         Metadata metadata,
         ClientStreamTracer.Factory tracerFactory,
         int previousAttempts,
-        boolean isTransparentRetry) {
+        boolean isTransparentRetry,
+        boolean isHedging) {
       bufferSizeTracer =
           tracerFactory.newClientStreamTracer(STREAM_INFO, metadata);
       int actualPreviousRpcAttemptsInHeader = metadata.get(GRPC_PREVIOUS_RPC_ATTEMPTS) == null
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java
@@ -18,9 +18,11 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static io.grpc.internal.GrpcUtil.IMPLEMENTATION_VERSION;
+import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.HEDGE_BUCKETS;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.LATENCY_BUCKETS;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.RETRY_BUCKETS;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.SIZE_BUCKETS;
+import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.TRANSPARENT_RETRY_BUCKETS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
@@ -247,12 +249,37 @@ static OpenTelemetryMetricsResource createMetricInstruments(Meter meter,
           meter.histogramBuilder(
                   "grpc.client.call.retries")
               .setUnit("{retry}")
-              .setDescription("Number of retry attempts made during the client call")
+              .setDescription("Number of retries during the client call. "
+                  + "If there were no retries, 0 is not reported.")
               .ofLongs()
               .setExplicitBucketBoundariesAdvice(RETRY_BUCKETS)
               .build());
     }
 
+    if (isMetricEnabled("grpc.client.call.transparent_retries", enableMetrics, disableDefault)) {
+      builder.clientCallRetriesCounter(
+          meter.histogramBuilder(
+                  "grpc.client.call.transparent_retries")
+              .setUnit("{transparent_retry}")
+              .setDescription("Number of transparent retries during the client call. "
+                  + "If there were no transparent retries, 0 is not reported.")
+              .ofLongs()
+              .setExplicitBucketBoundariesAdvice(TRANSPARENT_RETRY_BUCKETS)
+              .build());
+    }
+
+    if (isMetricEnabled("grpc.client.call.hedges", enableMetrics, disableDefault)) {
+      builder.clientCallRetriesCounter(
+          meter.histogramBuilder(
+                  "grpc.client.call.hedges")
+              .setUnit("{hedge}")
+              .setDescription("Number of hedges during the client call. "
+                  + "If there were no hedges, 0 is not reported.")
+              .ofLongs()
+              .setExplicitBucketBoundariesAdvice(HEDGE_BUCKETS)
+              .build());
+    }
+
     if (isMetricEnabled("grpc.client.call.retry_delay", enableMetrics, disableDefault)) {
       builder.clientCallRetryDelayCounter(
           meter.histogramBuilder(
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java
@@ -20,7 +20,6 @@
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BACKEND_SERVICE_KEY;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.LOCALITY_KEY;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.METHOD_KEY;
-import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.RETRY_TYPE_KEY;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.STATUS_KEY;
 import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.TARGET_KEY;
 
@@ -45,7 +44,6 @@
 import io.grpc.Status;
 import io.grpc.Status.Code;
 import io.grpc.StreamTracer;
-import io.grpc.opentelemetry.internal.OpenTelemetryConstants;
 import io.opentelemetry.api.common.AttributesBuilder;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -299,6 +297,7 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory
     private long callLatencyNanos;
     private final Object lock = new Object();
     private final AtomicLong attemptsPerCall = new AtomicLong();
+    private final AtomicLong hedgedAttemptsPerCall = new AtomicLong();
     private final AtomicLong transparentRetriesPerCall = new AtomicLong();
     @GuardedBy("lock")
     private int activeStreams;
@@ -352,7 +351,10 @@ public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata metada
       }
       if (info.isTransparentRetry()) {
         transparentRetriesPerCall.incrementAndGet();
-      } else {
+      } else if (info.isHedging()) {
+        hedgedAttemptsPerCall.incrementAndGet();
+      }
+      else {
         attemptsPerCall.incrementAndGet();
       }
       return newClientTracer(info);
@@ -442,19 +444,30 @@ void recordFinishedCall() {
           retriesPerCall = attempts - 1;
         }
 
-        module.resource.clientCallRetriesCounter().record(
-            retriesPerCall,
-            baseAttributes.toBuilder()
-                .put(RETRY_TYPE_KEY, OpenTelemetryConstants.RetryType.RETRY.getValue())
-                .build()
-        );
+        if (retriesPerCall > 0) {
+          module.resource.clientCallRetriesCounter().record(retriesPerCall, baseAttributes);
+        }
+      }
+
+      // Hedge counts
+      if (module.resource.clientCallHedgesCounter() != null) {
 
+        long hedgesPerCall = 0;
+        long attempts = hedgedAttemptsPerCall.get();
+        if (attempts > 0) {
+          hedgesPerCall = attempts - 1;
+        }
+
+        if (hedgesPerCall > 0) {
+          module.resource.clientCallHedgesCounter().record(hedgesPerCall, baseAttributes);
+        }
+      }
+
+      // Transparent Retry counts
+      if (module.resource.clientCallTransparentRetriesCounter() != null
+          && transparentRetriesPerCall.get() > 0) {
         module.resource.clientCallRetriesCounter().record(
-            transparentRetriesPerCall.get(),
-            baseAttributes.toBuilder()
-                .put(RETRY_TYPE_KEY, OpenTelemetryConstants.RetryType.TRANSPARENT.getValue())
-                .build()
-        );
+            transparentRetriesPerCall.get(), baseAttributes);
       }
 
       // Retry delay
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java
@@ -44,6 +44,12 @@ abstract class OpenTelemetryMetricsResource {
   @Nullable
   abstract LongHistogram clientCallRetriesCounter();
 
+  @Nullable
+  abstract LongHistogram clientCallTransparentRetriesCounter();
+
+  @Nullable
+  abstract LongHistogram clientCallHedgesCounter();
+
   @Nullable
   abstract DoubleHistogram clientCallRetryDelayCounter();
 
@@ -80,6 +86,10 @@ abstract Builder clientTotalReceivedCompressedMessageSizeCounter(
 
     abstract Builder clientCallRetriesCounter(LongHistogram counter);
 
+    abstract Builder clientCallTransparentRetriesCounter(LongHistogram counter);
+
+    abstract Builder clientCallHedgesCounter(LongHistogram counter);
+
     abstract Builder clientCallRetryDelayCounter(DoubleHistogram counter);
 
     abstract Builder serverCallCountCounter(LongCounter counter);
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/internal/OpenTelemetryConstants.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/internal/OpenTelemetryConstants.java
@@ -36,9 +36,6 @@ public final class OpenTelemetryConstants {
   public static final AttributeKey<String> BACKEND_SERVICE_KEY =
       AttributeKey.stringKey("grpc.lb.backend_service");
 
-  public static final AttributeKey<String> RETRY_TYPE_KEY =
-      AttributeKey.stringKey("grpc.retry_type");
-
   public static final List<Double> LATENCY_BUCKETS =
       ImmutableList.of(
           0d,     0.00001d, 0.00005d, 0.0001d, 0.0003d, 0.0006d, 0.0008d, 0.001d, 0.002d,
@@ -52,24 +49,12 @@ public final class OpenTelemetryConstants {
           0L, 1024L, 2048L, 4096L, 16384L, 65536L, 262144L, 1048576L, 4194304L, 16777216L,
           67108864L, 268435456L, 1073741824L, 4294967296L);
 
-  public static final List<Long> RETRY_BUCKETS =
-      ImmutableList.of(0L, 1L, 2L, 3L, 4L, 5L, 10L, 100L, 1000L);
-
-  public enum RetryType {
-    RETRY("retry"),
-    HEDGE("hedge"),
-    TRANSPARENT("transparent");
+  public static final List<Long> RETRY_BUCKETS = ImmutableList.of(0L, 1L, 2L, 3L, 4L, 5L);
 
-    private final String value;
+  public static final List<Long> TRANSPARENT_RETRY_BUCKETS =
+      ImmutableList.of(0L, 1L, 2L, 3L, 4L, 5L, 10L);
 
-    RetryType(String value) {
-      this.value = value;
-    }
-
-    public String getValue() {
-      return value;
-    }
-  }
+  public static final List<Long> HEDGE_BUCKETS = ImmutableList.of(0L, 1L, 2L, 3L, 4L, 5L);
 
   private OpenTelemetryConstants() {
   }
