Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,19 @@ void recordHttpRequestDuration(
httpClientRequestDuration.record(durationSeconds, attributes, Context.current());
}

void recordTtfb(
Context otelContext, String operationName, double ttfbSeconds, String networkPath) {
if (ttfb == null) {
return;
}
Attributes attributes =
baseAttributes.toBuilder()
.put(KEY_GCP_STORAGE_OPERATION, operationName)
.put(KEY_NETWORK_PATH, networkPath)
.build();
ttfb.record(ttfbSeconds, attributes, otelContext);
}

@Nullable
static StorageMetricsContext createContext(
@Nullable StorageClientMetrics metrics, String operationName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import static java.util.Objects.requireNonNull;

import com.google.api.core.ApiClock;
import com.google.storage.v2.BidiWriteObjectRequest;
import com.google.storage.v2.WriteObjectRequest;
import io.grpc.Attributes;
import io.grpc.CallOptions;
import io.grpc.Channel;
Expand All @@ -34,6 +36,8 @@
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.jspecify.annotations.NullMarked;
import org.jspecify.annotations.Nullable;

Expand All @@ -58,11 +62,19 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
String rpcMethod = method.getFullMethodName();
StorageMetricsContext metricsCtx = StorageMetricsContext.from(callOptions);
Context otelContext = metricsCtx != null ? metricsCtx.getOtelContext() : Context.current();
String operationName =
metricsCtx != null
? metricsCtx.getOperationName()
: defaultOperationName(method.getBareMethodName());

return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
private final AtomicBoolean ttfbRecorded = new AtomicBoolean(false);
private final AtomicLong ttfbStartNanos = new AtomicLong();

@Override
public void start(Listener<RespT> responseListener, Metadata headers) {
long startTimeNanos = clock.nanoTime();
ttfbStartNanos.set(startTimeNanos);

super.start(
new SimpleForwardingClientCallListener<RespT>(responseListener) {
Expand All @@ -74,6 +86,18 @@ public void onHeaders(Metadata headers) {
super.onHeaders(headers);
}

@Override
public void onMessage(@Nullable RespT message) {
// The first response message from the server (including metadata-only
// responses for 0-byte objects) marks Time To First Byte (TTFB).
if (ttfbRecorded.compareAndSet(false, true)) {
double ttfbSeconds = (clock.nanoTime() - ttfbStartNanos.get()) / 1_000_000_000.0;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If sendMessage() updates ttfbStartNanos concurrently after clock.nanoTime() is evaluated here, the elapsed time can go negative and get dropped by OTel. Can we read ttfbStartNanos.get() before clock.nanoTime() and guard re-anchoring with an AtomicBoolean so only the first flush updates it?

String networkPath = detectNetworkPath(getAttributes());
metrics.recordTtfb(otelContext, operationName, ttfbSeconds, networkPath);
}
super.onMessage(message);
}

@Override
public void onClose(Status status, Metadata trailers) {
try {
Expand All @@ -94,9 +118,44 @@ public void onClose(Status status, Metadata trailers) {
},
headers);
}

@Override
public void sendMessage(ReqT message) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For non-finalizing resumable WriteObject chunks, finish_write is false and the client triggers the server response via halfClose(). Should we also override halfClose() for CLIENT_STREAMING calls to re-anchor ttfbStartNanos so upload time isn't included in TTFB?

if (!ttfbRecorded.get() && isWriteFlush(message)) {
ttfbStartNanos.set(clock.nanoTime());
}
super.sendMessage(message);
}
};
}

static String defaultOperationName(@Nullable String bareMethod) {
if (bareMethod == null || bareMethod.isEmpty()) {
return "UNKNOWN";
}
switch (bareMethod) {
case "BidiReadObject":
return "ReadObject";
case "BidiWriteObject":
case "StartResumableWrite":
case "QueryWriteStatus":
return "WriteObject";
default:
return bareMethod;
}
}

static boolean isWriteFlush(@Nullable Object message) {
if (message instanceof BidiWriteObjectRequest) {
BidiWriteObjectRequest req = (BidiWriteObjectRequest) message;
return req.getFlush() || req.getFinishWrite();
}
if (message instanceof WriteObjectRequest) {
return ((WriteObjectRequest) message).getFinishWrite();
}
return false;
}

static String detectNetworkPath(@Nullable Attributes callAttributes) {
SocketAddress remoteAddr =
callAttributes != null ? callAttributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR) : null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.api.client.http.HttpUnsuccessfulResponseHandler;
import com.google.api.core.ApiClock;
import com.google.common.collect.ImmutableSet;
import io.opentelemetry.context.Context;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
Expand All @@ -42,6 +43,10 @@
* recorded here end at response headers and exclude streaming body transfer (a follow-up is tracked
* for recording when the response body is closed). End-to-end download duration including body
* transfer is captured by {@code gcp.client.request.duration}.
*
* <p>{@code gcp.storage.client.ttfb} is recorded here for media download requests across all HTTP
* response status codes when response headers arrive (HTTP unary and upload TTFB are tracked for a
* follow-up).
*/
@NullMarked
final class StorageHttpMetricsInterceptor implements HttpRequestInitializer {
Expand Down Expand Up @@ -132,6 +137,16 @@ private void recordResponse(HttpRequest request, long startNanos, long endNanos,
? StorageErrorTypeMapper.mapHttpResponseCode(statusCode)
: StorageErrorTypeMapper.STATUS_OK;
recordDuration(request, startNanos, endNanos, statusCode, errorType);
// Evaluate isMediaDownloadRequest(request) at response time rather than in
// initialize(HttpRequest), because callers such as MediaHttpDownloader may append
// query parameters (e.g. alt=media) to request.getUrl() after buildGetRequest().
if (isMediaDownloadRequest(request)) {
metrics.recordTtfb(
Context.current(),
"ReadObject",
(endNanos - startNanos) / 1_000_000_000.0,
StorageClientMetrics.NETWORK_PATH_CLOUDPATH);
}
}

private void recordDuration(
Expand All @@ -145,6 +160,16 @@ private void recordDuration(
errorType);
}

static boolean isMediaDownloadRequest(HttpRequest request) {
if (!"GET".equalsIgnoreCase(request.getRequestMethod())) {
return false;
}
GenericUrl url = request.getUrl();
String path = url.getRawPath();
return (path != null && path.startsWith("/download/storage/v1/"))
|| "media".equals(url.getFirst("alt"));
}

static String extractUrlTemplate(GenericUrl url) {
String path = url.getRawPath();
if (path == null || path.isEmpty()) {
Expand Down
Loading
Loading