diff --git a/.github/run-package-tests.sh b/.github/run-package-tests.sh index a02cfa1dbe1b..3f9af4596a86 100644 --- a/.github/run-package-tests.sh +++ b/.github/run-package-tests.sh @@ -70,7 +70,8 @@ for DIR in ${DIRS}; do "PubSub,cloud-pubsub" "Storage,cloud-storage" "ShoppingCommonProtos,shopping-common-protos" - "GeoCommonProtos,geo-common-protos,0.1" + "GeoCommonProtos,geo-common-protos,0.1", + "Monitoring,cloud-monitoring" ) for i in "${PACKAGE_DEPENDENCIES[@]}"; do IFS="," read -r PKG_DIR PKG_NAME PKG_VERSION <<< "$i" diff --git a/Spanner/composer.json b/Spanner/composer.json index 3e6f677ce53a..3ae8892fb33a 100644 --- a/Spanner/composer.json +++ b/Spanner/composer.json @@ -7,7 +7,9 @@ "php": "^8.1", "ext-grpc": "*", "google/cloud-core": "^1.68", - "google/gax": "^1.40.0" + "google/gax": "^1.40.0", + "google/cloud-monitoring": "^2.2", + "open-telemetry/sdk": "^1.13" }, "require-dev": { "phpunit/phpunit": "^9.6", diff --git a/Spanner/src/Middleware/BuiltInMetricsAttemptMiddleware.php b/Spanner/src/Middleware/BuiltInMetricsAttemptMiddleware.php new file mode 100644 index 000000000000..1f6deac7adc1 --- /dev/null +++ b/Spanner/src/Middleware/BuiltInMetricsAttemptMiddleware.php @@ -0,0 +1,240 @@ +nextHandler = $nextHandler; + $this->attemptLatencyHistogram = $meter->createHistogram( + 'attempt_latencies', + 'ms', + 'The latency of an RPC attempt' + ); + $this->attemptCountCounter = $meter->createCounter( + 'attempt_count', + '1', + 'The number of RPC attempts' + ); + $this->attemptGfeHistogram = $meter->createHistogram( + 'gfe_latencies', + 'ms', + 'Latency between Google\'s network receiving an RPC and reading back the first byte of the response' + ); + $this->gfeConnectivityErrorCounter = $meter->createCounter( + 'gfe_connectivity_error_count', + '1', + 'Number of RPC attempts that failed to reach the GFE or returned no GFE headers' + ); + $this->clientId = $clientId; + $this->projectId = $projectId; + $this->clientName = $clientName; + } + + public function __invoke(Call $call, array $options) + { + $next = $this->nextHandler; + + $startTime = microtime(true); + + // In case that something else is using this callback, + // we take the original one and call it later. + $originalCallback = $options['metadataCallback'] ?? null; + + // This gets the metadata on an ok status meaning we can get the GFE latency header for unary calls + $options['metadataCallback'] = function ($metadata) use ($originalCallback, $call, $options) { + $this->recordGfeLatency($metadata, $call, $options); + if ($originalCallback) { + $originalCallback($metadata); + } + }; + + try { + $response = $next( + $call, + $options + ); + } catch (Exception $e) { + // In case that the call is not a unary call and it is a streaming call error. + $this->recordAttempt($startTime, $e->getCode(), $call->getMethod(), $options); + $this->recordGfeError($e, $call, $options); + throw $e; + } + + if ($response instanceof ServerStream) { + $this->recordAttempt($startTime, Code::OK, $call->getMethod(), $options); + $this->recordGfeLatency($response->getServerStreamingCall()->getMetadata(), $call, $options); + } + + if ($response instanceof PromiseInterface) { + return $response->then( + function ($response) use ($startTime, $options, $call) { + $this->recordAttempt($startTime, Code::OK, $call->getMethod(), $options); + return $response; + }, + function ($e) use ($startTime, $options, $call) { + $this->recordAttempt($startTime, $e->getCode(), $call->getMethod(), $options); + $this->recordGfeError($e, $call, $options); + throw $e; + } + ); + } + + // The response can be a stream + return $response; + } + + /** + * Records an Attempt + * + * @param array $options The options being used for the middleware layer to communicate amongst middlewares + * @param float $startTime The start time of the RPC attempt + * @param int $code The resulting code of the attempt + * @param string $method The RPC method name that is being called + * + * @return void + */ + private function recordAttempt(float $startTime, int $code, string $method, array $options): void + { + $endTime = microtime(true); + $duration = ($endTime - $startTime) * 1000; // Convert to MS + + $labels = $this->getMetricLabels($method, $options, $code); + + $this->attemptCountCounter->add(1, $labels); + $this->attemptLatencyHistogram->record($duration, $labels); + } + + private function recordGfeLatency($metadata, Call $call, array $options): void + { + $serverTiming = $metadata['server-timing'][0] ?? null; + $gfeLatency = null; + + if ($serverTiming) { + if (preg_match('/gfet4t7;\s*dur=(\d+(\.\d+)?)/', $serverTiming, $matches)) { + $gfeLatency = (float) $matches[1]; + } + } + + $labels = $this->getMetricLabels($call->getMethod(), $options, Code::OK); + + if ($gfeLatency !== null) { + $this->attemptGfeHistogram->record($gfeLatency, $labels); + } else { + $this->gfeConnectivityErrorCounter->add(1, $labels); + } + } + + private function getMetricLabels(string $method, array $options, int $code): array + { + $codeName = Code::name($code); + + // Extract resource information from the GAX routing header. + $params = $options['headers']['x-goog-request-params'][0] ?? ''; + $prefix = urldecode($params); + + if (preg_match('/instances\/([^\/]+)\/databases\/([^\/]+)/', $prefix, $matches)) { + $instanceId = $matches[1]; + $databaseId = $matches[2]; + } + + return [ + 'method' => $method, + 'status' => $codeName, + 'instance_id' => $instanceId ?? '', + 'database' => $databaseId ?? '', + 'project_id' => $this->projectId, + 'client_uid' => $this->clientId, + 'client_name' => $this->clientName, + 'instance_config' => self::INSTANCE_CONFIG, + 'location' => self::LOCATION_LABEL + ]; + } + + private function recordGfeError(Exception $e, Call $call, array $options): void + { + if ($e instanceof ApiException) { + $this->recordGfeLatency($e->getMetadata() ?? [], $call, $options); + } else { + $this->recordGfeLatency([], $call, $options); + } + } +} diff --git a/Spanner/src/Middleware/BuiltInMetricsOperationMiddleware.php b/Spanner/src/Middleware/BuiltInMetricsOperationMiddleware.php new file mode 100644 index 000000000000..ef9bed1bb515 --- /dev/null +++ b/Spanner/src/Middleware/BuiltInMetricsOperationMiddleware.php @@ -0,0 +1,172 @@ +nextHandler = $nextHandler; + $this->operationLatencyHistogram = $meter->createHistogram( + 'operation_latencies', + 'ms', + 'The latency of an RPC operations' + ); + $this->operationCountCounter = $meter->createCounter( + 'operation_count', + '1', + 'The number of RPC operations' + ); + $this->clientId = $clientId; + $this->projectId = $projectId; + $this->clientName = $clientName; + } + + public function __invoke(Call $call, array $options) + { + $next = $this->nextHandler; + $startTime = microtime(true); + + try { + $response = $next( + $call, + $options + ); + } catch (Exception $ex) { + $this->recordOperation($startTime, $ex->getCode(), $call->getMethod(), $options); + throw $ex; + } + + if ($response instanceof ServerStream) { + $this->recordOperation($startTime, Code::OK, $call->getMethod(), $options); + } + + if ($response instanceof PromiseInterface) { + return $response->then( + function ($response) use ($startTime, $options, $call) { + $this->recordOperation($startTime, Code::OK, $call->getMethod(), $options); + return $response; + }, + function ($e) use ($startTime, $options, $call) { + $this->recordOperation($startTime, $e->getCode(), $call->getMethod(), $options); + throw $e; + } + ); + } + + // response can be a stream + return $response; + } + + /** + * Records a completed operation (failures are considered completions). + * + * @param float $startTime The start time of the operation + * @param int $code The resulting code of the operation + * @param string $method The RPC name being called + * @param array $options The options used for middleware communication + * + * @return void + */ + private function recordOperation(float $startTime, int $code, string $method, array $options): void + { + $endTime = microtime(true); + $duration = ($endTime - $startTime) * 1000; // Convert seconds to ms + $codeName = Code::name($code); + + // Extract resource information from the GAX routing header. + $params = $options['headers']['x-goog-request-params'][0] ?? ''; + $prefix = urldecode($params); + + if (preg_match('/instances\/([^\/]+)\/databases\/([^\/]+)/', $prefix, $matches)) { + $instanceId = $matches[1]; + $databaseId = $matches[2]; + } + + $labels = [ + 'method' => $method, + 'status' => $codeName, + 'instance_id' => $instanceId ?? '', + 'database' => $databaseId ?? '', + 'project_id' => $this->projectId, + 'client_uid' => $this->clientId, + 'client_name' => $this->clientName, + 'instance_config' => self::INSTANCE_CONFIG, + 'location' => self::LOCATION_LABEL + ]; + + $this->operationCountCounter->add(1, $labels); + $this->operationLatencyHistogram->record($duration, $labels); + } +} diff --git a/Spanner/src/OpenTelemetry/BuiltInMetricsExporter.php b/Spanner/src/OpenTelemetry/BuiltInMetricsExporter.php new file mode 100644 index 000000000000..656fd04bd40f --- /dev/null +++ b/Spanner/src/OpenTelemetry/BuiltInMetricsExporter.php @@ -0,0 +1,317 @@ + true, + 'instance_id' => true, + 'instance_config' => true, + 'location' => true, + 'client_hash' => true, + ]; + + private MetricServiceClient $client; + private string $projectId; + private string $clientHash; + + /** + * @param MetricServiceClient $client The monitoring client. + * @param string $projectId The GCP project ID metrics will be written to. + * @param string $clientUid The unique client identifier. + */ + public function __construct(MetricServiceClient $client, string $projectId, string $clientUid) + { + $this->client = $client; + $this->projectId = $projectId; + $this->clientHash = $this->generateClientHash($clientUid); + } + + /** + * Exports a batch of OTel metrics to Cloud Monitoring. + * + * @param iterable $batch + * @return bool + */ + public function export(iterable $batch): bool + { + $timeSeriesList = []; + foreach ($batch as $otelMetric) { + $timeSeriesList = array_merge($timeSeriesList, $this->mapMetric($otelMetric)); + } + + if (empty($timeSeriesList)) { + return true; + } + + $projectName = MetricServiceClient::projectName($this->projectId); + $chunks = array_chunk($timeSeriesList, self::SEND_BATCH_SIZE); + + foreach ($chunks as $chunk) { + $request = new CreateTimeSeriesRequest(); + $request->setName($projectName); + $request->setTimeSeries($chunk); + + try { + $this->client->createServiceTimeSeries($request); + } catch (\Exception $e) { + // Fail silently during shutdown to avoid user-visible errors. + } + } + + return true; + } + + /** + * Implementation of the forcePush method for PushMetricExporter interface. + * + * @return true + */ + public function forceFlush(): bool + { + return true; + } + + /** + * Implementation of the shutdown method for PushMetricExporterInterface. + * + * @return true + */ + public function shutdown(): bool + { + $this->client->close(); + return true; + } + + /** + * Returns the aggregation temporality for the given metric. + * + * @param MetricMetadataInterface $metadata + * @return string + */ + public function temporality(MetricMetadataInterface $metadata): string + { + return Temporality::CUMULATIVE; + } + + /** + * Maps an OTel Metric object to one or more GCM TimeSeries objects. + * + * @param OTelMetric $otelMetric + */ + private function mapMetric(OTelMetric $otelMetric): array + { + $timeSeriesList = []; + $metricType = $this->formatMetricName($otelMetric->name); + + $data = $otelMetric->data; + if ($data instanceof Sum || $data instanceof Histogram) { + foreach ($data->dataPoints as $point) { + $timeSeriesList[] = $this->createTimeSeries($metricType, $point, $otelMetric->unit, $data); + } + } + + return $timeSeriesList; + } + + /** + * Creates a single GCM TimeSeries from an OTel DataPoint. + * + * @param string $metricType + * @param NumberDataPoint|HistogramDataPoint $otelPoint + * @param string|null $unit + * @param DataInterface $otelData + * @return TimeSeries + */ + private function createTimeSeries( + string $metricType, + NumberDataPoint|HistogramDataPoint $otelPoint, + ?string $unit, + DataInterface $otelData + ): TimeSeries { + $ts = new TimeSeries(); + $unit = $unit ?? '1'; + + $metricLabels = []; + $resourceLabels = [ + 'client_hash' => $this->clientHash, + ]; + + // Distribute attributes between Resource and Metric labels + foreach ($otelPoint->attributes as $key => $value) { + $labelKey = str_replace('.', '_', $key); + if (isset(self::$MONITORED_RES_LABELS[$labelKey])) { + $resourceLabels[$labelKey] = (string) $value; + } else { + $metricLabels[$labelKey] = (string) $value; + } + } + + $metric = new Metric(); + $metric->setType($metricType); + $metric->setLabels($metricLabels); + $ts->setMetric($metric); + + $resource = new MonitoredResource(); + $resource->setType(self::SPANNER_RESOURCE_TYPE); + $resource->setLabels($resourceLabels); + $ts->setResource($resource); + + $ts->setUnit($unit); + + $point = new Point(); + $interval = new TimeInterval(); + + // Convert nanoseconds to Protobuf Timestamp + $interval->setStartTime($this->toTimestamp($otelPoint->startTimestamp)); + $interval->setEndTime($this->toTimestamp($otelPoint->timestamp)); + $point->setInterval($interval); + + $value = new TypedValue(); + if ($otelData instanceof Sum) { + $ts->setMetricKind($otelData->monotonic ? MetricKind::CUMULATIVE : MetricKind::GAUGE); + if (is_int($otelPoint->value)) { + $value->setInt64Value($otelPoint->value); + $ts->setValueType(ValueType::INT64); + } else { + $value->setDoubleValue((float) $otelPoint->value); + $ts->setValueType(ValueType::DOUBLE); + } + } elseif ($otelData instanceof Histogram) { + $ts->setMetricKind(MetricKind::CUMULATIVE); + $ts->setValueType(ValueType::DISTRIBUTION); + + $dist = new Distribution(); + $dist->setCount($otelPoint->count); + if ($otelPoint->count > 0) { + $dist->setMean($otelPoint->sum / $otelPoint->count); + } + $dist->setBucketCounts($otelPoint->bucketCounts); + + $bucketOptions = new BucketOptions(); + $explicit = new Explicit(); + $explicit->setBounds($otelPoint->explicitBounds); + $bucketOptions->setExplicitBuckets($explicit); + $dist->setBucketOptions($bucketOptions); + + $value->setDistributionValue($dist); + } + + $point->setValue($value); + $ts->setPoints([$point]); + + return $ts; + } + + /** + * Formats the metric name for Cloud Monitoring. + * Built-in metrics MUST use the specific internal namespace. + * + * @param string $name The OTel instrument name. + * @return string The fully qualified GCM metric type. + */ + private function formatMetricName(string $name): string + { + return self::NATIVE_METRICS_PREFIX . $name; + } + + /** + * Converts nanoseconds to a php Timestamp + * + * @param int $nanos + * @return Timestamp + */ + private function toTimestamp(int $nanos): Timestamp + { + $timestamp = new Timestamp(); + $timestamp->setSeconds((int) ($nanos / 1_000_000_000)); + $timestamp->setNanos((int) ($nanos % 1_000_000_000)); + return $timestamp; + } + + /** + * Returns a hash of the client UUID for the metrics + * + * @param string $clientUid + * @return string + */ + private function generateClientHash(string $clientUid): string + { + if ($clientUid === '') { + return '000000'; + } + + $hashHex = hash('fnv164', $clientUid); + $firstFour = substr($hashHex, 0, 4); + $intVal = hexdec($firstFour); + $tenBits = $intVal >> 6; + return sprintf('%06x', $tenBits); + } +} diff --git a/Spanner/src/SpannerClient.php b/Spanner/src/SpannerClient.php index dbe02bb165ad..24985e3a9e80 100644 --- a/Spanner/src/SpannerClient.php +++ b/Spanner/src/SpannerClient.php @@ -30,6 +30,7 @@ use Google\Cloud\Core\LongRunning\LongRunningClientConnection; use Google\Cloud\Core\LongRunning\LongRunningOperation; use Google\Cloud\Core\OptionsValidator; +use Google\Cloud\Monitoring\V3\Client\MetricServiceClient; use Google\Cloud\Spanner\Admin\Database\V1\Client\DatabaseAdminClient; use Google\Cloud\Spanner\Admin\Instance\V1\Client\InstanceAdminClient; use Google\Cloud\Spanner\Admin\Instance\V1\InstanceConfig; @@ -38,14 +39,23 @@ use Google\Cloud\Spanner\Admin\Instance\V1\ListInstancesRequest; use Google\Cloud\Spanner\Admin\Instance\V1\ReplicaInfo; use Google\Cloud\Spanner\Batch\BatchClient; +use Google\Cloud\Spanner\Middleware\BuiltInMetricsAttemptMiddleware; +use Google\Cloud\Spanner\Middleware\BuiltInMetricsOperationMiddleware; use Google\Cloud\Spanner\Middleware\RequestIdHeaderMiddleware; use Google\Cloud\Spanner\Middleware\SpannerMiddleware; +use Google\Cloud\Spanner\OpenTelemetry\BuiltInMetricsExporter; use Google\Cloud\Spanner\V1\Client\SpannerClient as GapicSpannerClient; use Google\Cloud\Spanner\V1\TransactionOptions\IsolationLevel; use Google\LongRunning\Operation as OperationProto; use Google\Protobuf\Duration; +use OpenTelemetry\API\Metrics\MeterInterface; +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use OpenTelemetry\SDK\Common\Util\ShutdownHandler; +use OpenTelemetry\SDK\Metrics\MeterProvider; +use OpenTelemetry\SDK\Metrics\MetricReader\ExportingReader; use Psr\Cache\CacheItemPoolInterface; use Psr\Http\Message\StreamInterface; +use Ramsey\Uuid\Uuid as RUUID; /** * Cloud Spanner is a highly scalable, transactional, managed, NewSQL @@ -133,6 +143,8 @@ class SpannerClient private array $defaultQueryOptions; private int $isolationLevel; private CacheItemPoolInterface|null $cacheItemPool; + private MeterInterface $meter; + private MeterProviderInterface $meterProvider; private static array $activeChannels = []; private static int $totalActiveChannels = 0; @@ -187,6 +199,12 @@ class SpannerClient * @type int $isolationLevel The level of Isolation for the transactions executed by this Client's instance. * **Defaults to** IsolationLevel::ISOLATION_LEVEL_UNSPECIFIED * @type CacheItemPoolInterface $cacheItemPool + * @type bool $disableBuiltInMetrics If true, built-in metrics collection will be disabled. + * **Defaults to** false. + * @type array $metricsOptions Configuration options for the internal `MetricServiceClient` + * used to export metrics. + * @type MetricServiceClient $metricServiceClient An explicit instance of + * `MetricServiceClient` to use for exporting metrics. * } * @throws GoogleException If the gRPC extension is not enabled. */ @@ -205,7 +223,8 @@ public function __construct(array $options = []) 'directedReadOptions' => [], 'isolationLevel' => IsolationLevel::ISOLATION_LEVEL_UNSPECIFIED, 'routeToLeader' => true, - 'cacheItemPool' => null + 'cacheItemPool' => null, + 'disableBuiltInMetrics' => false, ]; $this->returnInt64AsObject = $options['returnInt64AsObject']; @@ -274,6 +293,8 @@ public function __construct(array $options = []) $this->instanceAdminClient->addMiddleware($middleware); $this->databaseAdminClient->addMiddleware($middleware); + $this->configureBuiltinMetrics($options); + $this->projectName = InstanceAdminClient::projectName($this->projectId); $this->cacheItemPool = $options['cacheItemPool']; } @@ -1024,4 +1045,70 @@ private function configureKeepAlive(array $config): array return $config; } + + private function configureBuiltinMetrics(array &$options): void + { + $metricsClient = $this->pluck('metricServiceClient', $options, false); + $metricsOptions = $this->pluck('metricsOptions', $options, false) ?: []; + + if ($this->pluck('disableBuiltInMetrics', $options, false)) { + return; + } + + if (!$metricsClient) { + $metricsOptions += [ + 'projectId' => $this->projectId, + 'keyFile' => $options['keyFile'] ?? null, + 'keyFilePath' => $options['keyFilePath'] ?? null, + 'credentials' => $options['credentials'] ?? null, + 'credentialsConfig' => $options['credentialsConfig'] ?? null, + 'universeDomain' => $options['universeDomain'] ?? null, + 'transport' => $options['transport'] ?? null, + 'transportConfig' => $options['transportConfig'] ?? null, + ]; + + try { + $metricsClient = new MetricServiceClient($metricsOptions); + } catch (ValidationException $e) { + return; + } + } + + if (!$metricsClient instanceof MetricServiceClient) { + throw new ValidationException('The "metricServiceClient" option must be a MetricServiceClient instance.'); + } + + $metricsClientId = RUUID::uuid4()->toString() . '-' . getmypid(); + $exporter = new BuiltInMetricsExporter($metricsClient, $this->projectId, $metricsClientId); + $reader = new ExportingReader($exporter); + $this->meterProvider = MeterProvider::builder() + ->addReader($reader) + ->build(); + + $this->meter = $this->meterProvider->getMeter('google-cloud-spanner'); + ShutdownHandler::register([$this->meterProvider, 'shutdown']); + + $attemptMetricsMiddleware = function (MiddlewareInterface $handler) use ($metricsClientId) { + return new BuiltInMetricsAttemptMiddleware( + $handler, + $this->meter, + $metricsClientId, + $this->projectId, + SpannerClient::VERSION + ); + }; + + $operationMetricsMiddleware = function (MiddlewareInterface $handler) use ($metricsClientId) { + return new BuiltInMetricsOperationMiddleware( + $handler, + $this->meter, + $metricsClientId, + $this->projectId, + SpannerClient::VERSION + ); + }; + + $this->spannerClient->prependMiddleware($attemptMetricsMiddleware); + $this->spannerClient->addMiddleware($operationMetricsMiddleware); + } } diff --git a/Spanner/tests/Snippet/CommitTimestampTest.php b/Spanner/tests/Snippet/CommitTimestampTest.php index 4b7d7a0ef09b..d6fd7ec71d5f 100644 --- a/Spanner/tests/Snippet/CommitTimestampTest.php +++ b/Spanner/tests/Snippet/CommitTimestampTest.php @@ -57,10 +57,13 @@ public function testClass() { $id = 'abc'; + // One add for the SpannerMiddleware and one for the Metrics middleware $this->spannerClient->addMiddleware(Argument::type('callable')) - ->shouldBeCalledOnce(); + ->shouldBeCalled(2); + + // One prepend for the Spanner Header Id and one for the Metrics middleware $this->spannerClient->prependMiddleware(Argument::type('callable')) - ->shouldBeCalledOnce(); + ->shouldBeCalled(2); // ensure cache hit $cacheItem = $this->prophesize(CacheItemInterface::class); diff --git a/Spanner/tests/System/QueryTest.php b/Spanner/tests/System/QueryTest.php index 4aa5a2b45de8..3fe4723e2829 100644 --- a/Spanner/tests/System/QueryTest.php +++ b/Spanner/tests/System/QueryTest.php @@ -27,6 +27,7 @@ use Google\Cloud\Spanner\Interval; use Google\Cloud\Spanner\Numeric; use Google\Cloud\Spanner\Result; +use Google\Cloud\Spanner\SpannerClient; use Google\Cloud\Spanner\StructType; use Google\Cloud\Spanner\StructValue; use Google\Cloud\Spanner\Timestamp; @@ -1251,4 +1252,33 @@ public function testBindStructInferredParameterTypesWithUnnamed() ] ], $res); } + + /** + * This test ensures that enabling built-in metrics does not interfere with + * normal client operations and that the OpenTelemetry pipeline is stable. + */ + public function testBuiltInMetrics() + { + if (self::isEmulatorUsed()) { + $this->markTestSkipped('Built-in metrics are not supported on Emulator.'); + } + + $keyFilePath = getenv('GOOGLE_CLOUD_PHP_TESTS_KEY_PATH'); + $client = new SpannerClient([ + 'keyFilePath' => $keyFilePath, + 'disableBuiltInMetrics' => false + ]); + + $db = $client->connect(self::INSTANCE_NAME, self::$dbName); + + // Execute a query to trigger metrics (Attempt, Operation, GFE) + $res = $db->execute('SELECT 1'); + $row = $res->rows()->current(); + + $this->assertEquals(1, $row[0]); + + // Success here means the middlewares correctly handled the request/response + // and recorded metrics without throwing exceptions. + // The final export happens at PHP shutdown via ShutdownHandler. + } } diff --git a/Spanner/tests/System/SystemTestCaseTrait.php b/Spanner/tests/System/SystemTestCaseTrait.php index ab0e8ac4a3ab..87d5ef1ec284 100644 --- a/Spanner/tests/System/SystemTestCaseTrait.php +++ b/Spanner/tests/System/SystemTestCaseTrait.php @@ -67,9 +67,9 @@ private static function getClient() ], ] ]; - $clientConfig = [ 'keyFilePath' => $keyFilePath, + 'disableBuiltInMetrics' => true, ]; $serviceAddress = getenv('SPANNER_SERVICE_ADDRESS'); diff --git a/Spanner/tests/Unit/Middleware/BuiltInMetricsAttemptMiddlewareTest.php b/Spanner/tests/Unit/Middleware/BuiltInMetricsAttemptMiddlewareTest.php new file mode 100644 index 000000000000..9bac17339184 --- /dev/null +++ b/Spanner/tests/Unit/Middleware/BuiltInMetricsAttemptMiddlewareTest.php @@ -0,0 +1,238 @@ +attemptHistogram = $this->prophesize(HistogramInterface::class); + $this->attemptCounter = $this->prophesize(CounterInterface::class); + $this->gfeHistogram = $this->prophesize(HistogramInterface::class); + $this->gfeErrorCounter = $this->prophesize(CounterInterface::class); + $this->meter = $this->prophesize(MeterInterface::class); + + $this->meter->createHistogram( + 'attempt_latencies', + 'ms', + Argument::any() + )->willReturn($this->attemptHistogram->reveal()); + + $this->meter->createCounter( + 'attempt_count', + '1', + Argument::any() + )->willReturn($this->attemptCounter->reveal()); + + $this->meter->createHistogram( + 'gfe_latencies', + 'ms', + Argument::any() + )->willReturn($this->gfeHistogram->reveal()); + + $this->meter->createCounter( + 'gfe_connectivity_error_count', + '1', + Argument::any() + )->willReturn($this->gfeErrorCounter->reveal()); + + $this->nextHandler = function ($call, $options) { + if (isset($options['metadataCallback'])) { + $options['metadataCallback'](['server-timing' => ['gfet4t7; dur=12.5']]); + } + return new FulfilledPromise('ok'); + }; + } + + public function testRecordsAttemptMetrics() + { + $projectId = 'test-project'; + $clientId = 'test-client-id'; + $clientName = 'php-spanner/1.0.0'; + + $middleware = new BuiltInMetricsAttemptMiddleware( + $this->nextHandler, + $this->meter->reveal(), + $clientId, + $projectId, + $clientName + ); + + $call = $this->prophesize(Call::class); + $call->getMethod()->willReturn('Commit'); + + // GAX formats this as a URL-encoded string in a header + $options = [ + 'headers' => [ + 'x-goog-request-params' => ['database=projects%2Fp%2Finstances%2Fi%2Fdatabases%2Fd'] + ] + ]; + + // Verify Labels + $expectedLabels = [ + 'method' => 'Commit', + 'status' => 'OK', + 'instance_id' => 'i', + 'database' => 'd', + 'project_id' => $projectId, + 'client_uid' => $clientId, + 'client_name' => $clientName, + 'instance_config' => 'unknown', + 'location' => 'global' + ]; + + $this->attemptCounter->add(1, $expectedLabels)->shouldBeCalled(); + $this->attemptHistogram->record(Argument::type('float'), $expectedLabels)->shouldBeCalled(); + + // GFE metrics + $this->gfeHistogram->record(12.5, $expectedLabels)->shouldBeCalled(); + + $promise = $middleware($call->reveal(), $options); + $promise->wait(); + } + + public function testRecordsGfeMetricsOnStreamingResponse() + { + $callWrapper = $this->prophesize(ServerStreamingCallInterface::class); + $callWrapper->getMetadata()->willReturn(['server-timing' => ['gfet4t7; dur=45.0']]); + + $serverStream = new ServerStream($callWrapper->reveal()); + + $this->nextHandler = function ($call, $options) use ($serverStream) { + return $serverStream; + }; + + $middleware = new BuiltInMetricsAttemptMiddleware( + $this->nextHandler, + $this->meter->reveal(), + 'client', + 'project', + 'name' + ); + + $call = $this->prophesize(Call::class); + $call->getMethod()->willReturn('ExecuteStreamingSql'); + + $options = [ + 'headers' => [ + 'x-goog-request-params' => ['database=projects%2Fp%2Finstances%2Fi%2Fdatabases%2Fd'] + ] + ]; + + // Expect GFE latency recording from the stream metadata + $this->gfeHistogram->record(45.0, Argument::any())->shouldBeCalled(); + $this->attemptCounter->add(1, Argument::any())->shouldBeCalled(); + + $middleware($call->reveal(), $options); + } + + public function testRecordsGfeErrorOnMissingHeader() + { + $this->nextHandler = function ($call, $options) { + if (isset($options['metadataCallback'])) { + $options['metadataCallback']([]); // Missing header + } + return new FulfilledPromise('ok'); + }; + + $middleware = new BuiltInMetricsAttemptMiddleware( + $this->nextHandler, + $this->meter->reveal(), + 'client', + 'project', + 'name' + ); + + $call = $this->prophesize(Call::class); + $call->getMethod()->willReturn('Commit'); + + $options = [ + 'headers' => [ + 'x-goog-request-params' => ['database=projects%2Fp%2Finstances%2Fi%2Fdatabases%2Fd'] + ] + ]; + + $this->gfeErrorCounter->add(1, Argument::any())->shouldBeCalled(); + + $promise = $middleware($call->reveal(), $options); + $promise->wait(); + } + + public function testRecordsMetricsOnError() + { + $this->nextHandler = function ($call, $options) { + return new RejectedPromise(new \Exception('fail', 7)); + }; + + $middleware = new BuiltInMetricsAttemptMiddleware( + $this->nextHandler, + $this->meter->reveal(), + 'client', + 'project', + 'name' + ); + + $call = $this->prophesize(Call::class); + $call->getMethod()->willReturn('Commit'); + + $options = [ + 'headers' => [ + 'x-goog-request-params' => ['database=projects%2Fp%2Finstances%2Fi%2Fdatabases%2Fd'] + ] + ]; + + // On error, we expect attempt count/latency AND GFE error count (since headers were missing) + $this->attemptCounter->add(1, Argument::any())->shouldBeCalled(); + $this->attemptHistogram->record(Argument::any(), Argument::any())->shouldBeCalled(); + $this->gfeErrorCounter->add(1, Argument::any())->shouldBeCalled(); + + $promise = $middleware($call->reveal(), $options); + + try { + $promise->wait(); + } catch (\Exception $e) { + $this->assertEquals(7, $e->getCode()); + } + } +} diff --git a/Spanner/tests/Unit/Middleware/BuiltInMetricsOperationMiddlewareTest.php b/Spanner/tests/Unit/Middleware/BuiltInMetricsOperationMiddlewareTest.php new file mode 100644 index 000000000000..16b8f179fc16 --- /dev/null +++ b/Spanner/tests/Unit/Middleware/BuiltInMetricsOperationMiddlewareTest.php @@ -0,0 +1,107 @@ +histogram = $this->prophesize(HistogramInterface::class); + $this->counter = $this->prophesize(CounterInterface::class); + $this->meter = $this->prophesize(MeterInterface::class); + + $this->meter->createHistogram( + 'operation_latencies', + 'ms', + Argument::any() + )->willReturn($this->histogram->reveal()); + + $this->meter->createCounter( + 'operation_count', + '1', + Argument::any() + )->willReturn($this->counter->reveal()); + + $this->nextHandler = function ($call, $options) { + return new FulfilledPromise('ok'); + }; + } + + public function testRecordsOperationMetrics() + { + $projectId = 'test-project'; + $clientId = 'test-client-id'; + $clientName = 'php-spanner/1.0.0'; + + $middleware = new BuiltInMetricsOperationMiddleware( + $this->nextHandler, + $this->meter->reveal(), + $clientId, + $projectId, + $clientName + ); + + $call = $this->prophesize(Call::class); + $call->getMethod()->willReturn('ExecuteSql'); + + $options = [ + 'headers' => [ + 'x-goog-request-params' => ['database=projects%2Fp%2Finstances%2Fi%2Fdatabases%2Fd'] + ] + ]; + + // Verify Labels + $expectedLabels = [ + 'method' => 'ExecuteSql', + 'status' => 'OK', + 'instance_id' => 'i', + 'database' => 'd', + 'project_id' => $projectId, + 'client_uid' => $clientId, + 'client_name' => $clientName, + 'instance_config' => 'unknown', + 'location' => 'global' + ]; + + $this->counter->add(1, $expectedLabels)->shouldBeCalled(); + $this->histogram->record(Argument::type('float'), $expectedLabels)->shouldBeCalled(); + + $promise = $middleware($call->reveal(), $options); + $promise->wait(); + } +} diff --git a/Spanner/tests/Unit/OpenTelemetry/BuiltInMetricsExporterTest.php b/Spanner/tests/Unit/OpenTelemetry/BuiltInMetricsExporterTest.php new file mode 100644 index 000000000000..34f269fadc2a --- /dev/null +++ b/Spanner/tests/Unit/OpenTelemetry/BuiltInMetricsExporterTest.php @@ -0,0 +1,139 @@ +prophesize(MetricServiceClient::class); + $exporter = new BuiltInMetricsExporter($client->reveal(), self::PROJECT_ID, self::CLIENT_ID); + + $reflection = new ReflectionClass(BuiltInMetricsExporter::class); + $method = $reflection->getMethod('generateClientHash'); + $method->setAccessible(true); + + $result = $method->invoke($exporter, $clientUid); + $this->assertEquals($expected, $result); + } + + public function hashDataProvider() + { + return [ + ['exampleUID', '00006b'], + ['', '000000'], + ['!@#$%^&*()', '000389'], + ['aVeryLongUniqueIdentifierThatExceedsNormalLength', '000125'], + ['1234567890', '00003e'], + ]; + } + + public function testExport() + { + $client = $this->prophesize(MetricServiceClient::class); + $exporter = new BuiltInMetricsExporter($client->reveal(), self::PROJECT_ID, self::CLIENT_ID); + + $scope = new InstrumentationScope('google-cloud-spanner', '1.0.0', null, Attributes::create([])); + $resource = ResourceInfo::create(Attributes::create(['service.name' => 'spanner'])); + + $attributes = Attributes::create([ + 'method' => 'ExecuteSql', + 'status' => 'OK', + 'instance_id' => 'my-instance', + 'database' => 'my-db' + ]); + + $point = new NumberDataPoint( + 1, + $attributes, + 1711368000000000000, // nanoseconds + 1711368060000000000 + ); + + $sum = new Sum([$point], Temporality::CUMULATIVE, true); + $metric = new OTelMetric($scope, $resource, 'attempt_count', '1', 'desc', $sum); + + $client->createServiceTimeSeries(Argument::that(function ($request) { + if (!$request instanceof CreateTimeSeriesRequest) { + return false; + } + + $projectName = MetricServiceClient::projectName(self::PROJECT_ID); + if ($request->getName() !== $projectName) { + return false; + } + + $timeSeries = $request->getTimeSeries()[0]; + + // Verify Metric Type + $expectedMetric = 'spanner.googleapis.com/internal/client/attempt_count'; + if ($timeSeries->getMetric()->getType() !== $expectedMetric) { + return false; + } + + // Verify Labels + $labels = $timeSeries->getMetric()->getLabels(); + if ($labels['method'] !== 'ExecuteSql' || + $labels['status'] !== 'OK' || + $labels['database'] !== 'my-db') { + return false; + } + + // Verify Resource + $resLabels = $timeSeries->getResource()->getLabels(); + if ($resLabels['instance_id'] !== 'my-instance') { + return false; + } + + // Verify Client Hash + if ($resLabels['client_hash'] !== '000369') { + return false; + } + + return true; + }), Argument::any())->shouldBeCalled(); + + $this->assertTrue($exporter->export([$metric])); + } +} diff --git a/Spanner/tests/Unit/SpannerClientTest.php b/Spanner/tests/Unit/SpannerClientTest.php index 7421d0de573a..435a70cfd44f 100644 --- a/Spanner/tests/Unit/SpannerClientTest.php +++ b/Spanner/tests/Unit/SpannerClientTest.php @@ -800,4 +800,35 @@ public function testConfigureKeepAlive() $newConfig['transportConfig']['grpc']['stubOpts']['grpc.keepalive_time_ms'] ); } + + public function testBuiltinMetricsEnabledByDefault() + { + $gapicSpannerClient = $this->prophesize(GapicSpannerClient::class); + $gapicSpannerClient->prependMiddleware(Argument::any()) + ->shouldBeCalledTimes(2); + $gapicSpannerClient->addMiddleware(Argument::any()) + ->shouldBeCalledTimes(2); + + new SpannerClient([ + 'projectId' => self::PROJECT, + 'credentials' => Fixtures::KEYFILE_STUB_FIXTURE(), + 'gapicSpannerClient' => $gapicSpannerClient->reveal(), + ]); + } + + public function testBuiltinMetricsCanBeDisabled() + { + $gapicSpannerClient = $this->prophesize(GapicSpannerClient::class); + $gapicSpannerClient->prependMiddleware(Argument::any()) + ->shouldBeCalledTimes(1); + $gapicSpannerClient->addMiddleware(Argument::any()) + ->shouldBeCalledTimes(1); + + new SpannerClient([ + 'projectId' => self::PROJECT, + 'credentials' => Fixtures::KEYFILE_STUB_FIXTURE(), + 'gapicSpannerClient' => $gapicSpannerClient->reveal(), + 'disableBuiltInMetrics' => true, + ]); + } } diff --git a/Spanner/tests/Unit/bootstrap.php b/Spanner/tests/Unit/bootstrap.php index f16f16a2c9c5..7973127ece79 100644 --- a/Spanner/tests/Unit/bootstrap.php +++ b/Spanner/tests/Unit/bootstrap.php @@ -7,6 +7,7 @@ '*/src/Admin/Database/V1/Client/*', '*/src/Admin/Instance/V1/Client/*', '*/src/V1/Client/*', + '*/vendor/google/cloud-monitoring/src/V3/Client/*' ]); BypassFinals::enable(); diff --git a/composer.json b/composer.json index 9ffd11de6347..2cf56812cac4 100644 --- a/composer.json +++ b/composer.json @@ -65,7 +65,8 @@ "ramsey/uuid": "^4.0", "google/common-protos": "^4.4", "google/gax": "^1.40.0", - "google/auth": "^1.42" + "google/auth": "^1.42", + "open-telemetry/sdk": "^1.13" }, "require-dev": { "phpunit/phpunit": "^9.6",