protobuf: move from OpenMetrics to Prometheus proto

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.25.0]
+
+### Added
+
+- Added an off-by-default `protobuf-protox` feature to build protobuf support
+  without requiring an external `protoc` binary.
+
+### Changed
+
+- Updated `prost`, `prost-build`, and `prost-types` dependencies to `v0.14`.
+- The `protobuf` feature now generates and encodes Prometheus
+  `io.prometheus.client` protobuf messages from `metrics.proto` rather than the
+  OpenMetrics protobuf data model. This is a breaking change for users of the `protobuf` feature.
+  See [Issue](https://github.com/prometheus/OpenMetrics/issues/296) for more context.
+
 ## [0.24.1]
 
 ### Added

CONTRIBUTING.md

@@ -6,6 +6,9 @@ The `build.rs` script in this library depends upon the
 [Protocol Buffers compiler][protoc]. Be sure that `protoc` is installed and
 available within your `PATH`.
 
+If you enable the off-by-default `protobuf-protox` feature, the build uses
+`protox` instead and does not require `protoc`.
+
 [protoc]: https://docs.rs/prost-build/latest/prost_build/#sourcing-protoc
 
 ## Python Dependencies

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "prometheus-client"
-version = "0.24.1"
+version = "0.25.0"
 authors = ["Max Inden <mail@max-inden.de>"]
 edition = "2021"
 description = "Open Metrics client library allowing users to natively instrument applications."
@@ -13,6 +13,7 @@ documentation = "https://docs.rs/prometheus-client"
 [features]
 default = []
 protobuf = ["dep:prost", "dep:prost-types", "dep:prost-build"]
+protobuf-protox = ["protobuf", "dep:protox"]
 
 # This feature provides additional APIs for testing downstream code using
 # `prometheus-client`.
@@ -29,8 +30,8 @@ dtoa = "1.0"
 itoa = "1.0"
 parking_lot = "0.12"
 prometheus-client-derive-encode = { version = "0.5.0", path = "derive-encode" }
-prost = { version = "0.12.0", optional = true }
-prost-types = { version = "0.12.0", optional = true }
+prost = { version = "0.14", optional = true }
+prost-types = { version = "0.14", optional = true }
 
 [dev-dependencies]
 async-std = { version = "1", features = ["attributes"] }
@@ -49,7 +50,8 @@ hyper-util = { version = "0.1.3", features = ["tokio"] }
 http-body-util = "0.1.1"
 
 [build-dependencies]
-prost-build = { version = "0.12.0", optional = true }
+prost-build = { version = "0.14", optional = true }
+protox = { version = "0.9.1", optional = true }
 
 [[bench]]
 name = "baseline"

build.rs

@@ -1,11 +1,29 @@
-use std::io::Result;
+use std::error::Error;
 
-fn main() -> Result<()> {
+fn main() -> Result<(), Box<dyn Error>> {
     #[cfg(feature = "protobuf")]
-    prost_build::compile_protos(
-        &["src/encoding/proto/openmetrics_data_model.proto"],
-        &["src/encoding/proto/"],
-    )?;
+    compile_protos()?;
+
+    Ok(())
+}
+
+#[cfg(feature = "protobuf")]
+fn compile_protos() -> Result<(), Box<dyn Error>> {
+    let protos = ["src/encoding/proto/metrics.proto"];
+    let includes = ["src/encoding/proto/"];
+
+    #[cfg(feature = "protobuf-protox")]
+    prost_build::compile_fds(protox::compile(protos, includes)?)?;
+
+    #[cfg(not(feature = "protobuf-protox"))]
+    prost_build::compile_protos(&protos, &includes)?;
+
+    for path in &protos {
+        println!("cargo:rerun-if-changed={}", path);
+    }
+    for path in &includes {
+        println!("cargo:rerun-if-changed={}", path);
+    }
 
     Ok(())
 }

derive-encode/tests/lib.rs

@@ -48,7 +48,7 @@ fn basic_flow() {
 mod protobuf {
     use crate::{Labels, Method};
     use prometheus_client::encoding::protobuf::encode;
-    use prometheus_client::encoding::protobuf::openmetrics_data_model;
+    use prometheus_client::encoding::protobuf::prometheus_data_model;
     use prometheus_client::metrics::counter::Counter;
     use prometheus_client::metrics::family::Family;
     use prometheus_client::registry::Registry;
@@ -67,17 +67,15 @@ mod protobuf {
             })
             .inc();
 
-        // Encode all metrics in the registry in the OpenMetrics protobuf format.
-        let mut metric_set = encode(&registry).unwrap();
-        let mut family: openmetrics_data_model::MetricFamily =
-            metric_set.metric_families.pop().unwrap();
-        let metric: openmetrics_data_model::Metric = family.metrics.pop().unwrap();
+        let mut metric_families = encode(&registry).unwrap();
+        let mut family: prometheus_data_model::MetricFamily = metric_families.pop().unwrap();
+        let metric: prometheus_data_model::Metric = family.metric.pop().unwrap();
 
-        let method = &metric.labels[0];
+        let method = &metric.label[0];
         assert_eq!("method", method.name);
         assert_eq!("Get", method.value);
 
-        let path = &metric.labels[1];
+        let path = &metric.label[1];
         assert_eq!("path", path.name);
         assert_eq!("/metrics", path.value);
     }
@@ -96,13 +94,11 @@ mod protobuf {
             })
             .inc();
 
-        // Encode all metrics in the registry in the OpenMetrics protobuf format.
-        let mut metric_set = encode(&registry).unwrap();
-        let mut family: openmetrics_data_model::MetricFamily =
-            metric_set.metric_families.pop().unwrap();
-        let metric: openmetrics_data_model::Metric = family.metrics.pop().unwrap();
+        let mut metric_families = encode(&registry).unwrap();
+        let mut family: prometheus_data_model::MetricFamily = metric_families.pop().unwrap();
+        let metric: prometheus_data_model::Metric = family.metric.pop().unwrap();
 
-        let label = &metric.labels[0];
+        let label = &metric.label[0];
         assert_eq!("method", label.name);
         assert_eq!("Get", label.value);
     }

src/encoding.rs

@@ -611,10 +611,7 @@ impl EncodeGaugeValue for i64 {
 
 impl EncodeGaugeValue for u64 {
     fn encode(&self, encoder: &mut GaugeValueEncoder) -> Result<(), std::fmt::Error> {
-        // Between forcing end users to do endless as i64 for things that are
-        // clearly valid i64 and having one error case for rarely used protobuf when
-        // a gauge is set to >i64::MAX, the latter seems like the right choice.
-        encoder.encode_i64(i64::try_from(*self).map_err(|_err| std::fmt::Error)?)
+        encoder.encode_u64(*self)
     }
 }
 
@@ -667,6 +664,10 @@ impl GaugeValueEncoder<'_> {
         for_both_mut!(self, GaugeValueEncoderInner, e, e.encode_u32(v))
     }
 
+    fn encode_u64(&mut self, v: u64) -> Result<(), std::fmt::Error> {
+        for_both_mut!(self, GaugeValueEncoderInner, e, e.encode_u64(v))
+    }
+
     fn encode_i64(&mut self, v: i64) -> Result<(), std::fmt::Error> {
         for_both_mut!(self, GaugeValueEncoderInner, e, e.encode_i64(v))
     }

src/encoding/proto/metrics.proto

@@ -0,0 +1,153 @@
+// Copyright 2013 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package io.prometheus.client;
+option go_package = "io_prometheus_client";
+
+import "google/protobuf/timestamp.proto";
+
+message LabelPair {
+  string name  = 1;
+  string value = 2;
+}
+
+enum MetricType {
+  // COUNTER must use the Metric field "counter".
+  COUNTER = 0;
+  // GAUGE must use the Metric field "gauge".
+  GAUGE = 1;
+  // SUMMARY must use the Metric field "summary".
+  SUMMARY = 2;
+  // UNTYPED must use the Metric field "untyped".
+  UNTYPED = 3;
+  // HISTOGRAM must use the Metric field "histogram".
+  HISTOGRAM = 4;
+  // GAUGE_HISTOGRAM must use the Metric field "histogram".
+  GAUGE_HISTOGRAM = 5;
+}
+
+message Gauge {
+  double value = 1;
+}
+
+message Counter {
+  double   value    = 1;
+  Exemplar exemplar = 2;
+
+  google.protobuf.Timestamp  start_timestamp = 3;
+}
+
+message Quantile {
+  double quantile = 1;
+  double value    = 2;
+}
+
+message Summary {
+  uint64            sample_count = 1;
+  double            sample_sum   = 2;
+  repeated Quantile quantile     = 3;
+
+  google.protobuf.Timestamp  start_timestamp = 4;
+}
+
+message Untyped {
+  double value = 1;
+}
+
+message Histogram {
+  uint64 sample_count       = 1;
+  double sample_count_float = 4; // Overrides sample_count if > 0.
+  double sample_sum         = 2;
+  // Buckets for the classic histogram.
+  repeated Bucket bucket = 3; // Ordered in increasing order of upper_bound, +Inf bucket is optional.
+
+  google.protobuf.Timestamp start_timestamp = 15;
+
+  // Everything below here is for native histograms (formerly known as sparse histograms).
+
+  // schema defines the bucket schema. Currently, valid numbers are -4 <= n <= 8.
+  // They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and
+  // then each power of two is divided into 2^n logarithmic buckets.
+  // Or in other words, each bucket boundary is the previous boundary times 2^(2^-n).
+  // In the future, more bucket schemas may be added using numbers < -4 or > 8.
+  sint32 schema           = 5;
+  double zero_threshold   = 6; // Breadth of the zero bucket.
+  uint64 zero_count       = 7; // Count in zero bucket.
+  double zero_count_float = 8; // Overrides sb_zero_count if > 0.
+
+  // Negative buckets for the native histogram.
+  repeated BucketSpan negative_span = 9;
+  // Use either "negative_delta" or "negative_count", the former for
+  // regular histograms with integer counts, the latter for float
+  // histograms.
+  repeated sint64 negative_delta = 10; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
+  repeated double negative_count = 11; // Absolute count of each bucket.
+
+  // Positive buckets for the native histogram.
+  // Use a no-op span (offset 0, length 0) for a native histogram without any
+  // observations yet and with a zero_threshold of 0. Otherwise, it would be
+  // indistinguishable from a classic histogram.
+  repeated BucketSpan positive_span = 12;
+  // Use either "positive_delta" or "positive_count", the former for
+  // regular histograms with integer counts, the latter for float
+  // histograms.
+  repeated sint64 positive_delta = 13; // Count delta of each bucket compared to previous one (or to zero for 1st bucket).
+  repeated double positive_count = 14; // Absolute count of each bucket.
+
+  // Only used for native histograms. These exemplars MUST have a timestamp.
+  repeated Exemplar exemplars = 16;
+}
+
+message Bucket {
+  uint64   cumulative_count       = 1; // Cumulative in increasing order.
+  double   cumulative_count_float = 4; // Overrides cumulative_count if > 0.
+  double   upper_bound            = 2; // Inclusive.
+  Exemplar exemplar               = 3;
+}
+
+// A BucketSpan defines a number of consecutive buckets in a native
+// histogram with their offset. Logically, it would be more
+// straightforward to include the bucket counts in the Span. However,
+// the protobuf representation is more compact in the way the data is
+// structured here (with all the buckets in a single array separate
+// from the Spans).
+message BucketSpan {
+  sint32 offset = 1; // Gap to previous span, or starting point for 1st span (which can be negative).
+  uint32 length = 2; // Length of consecutive buckets.
+}
+
+message Exemplar {
+  repeated LabelPair        label     = 1;
+  double                    value     = 2;
+  google.protobuf.Timestamp timestamp = 3; // OpenMetrics-style.
+}
+
+message Metric {
+  repeated LabelPair label        = 1;
+  Gauge              gauge        = 2;
+  Counter            counter      = 3;
+  Summary            summary      = 4;
+  Untyped            untyped      = 5;
+  Histogram          histogram    = 7;
+  int64              timestamp_ms = 6;
+}
+
+message MetricFamily {
+  string          name   = 1;
+  string          help   = 2;
+  MetricType      type   = 3;
+  repeated Metric metric = 4;
+  string          unit   = 5;
+}