diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index e52c65a8a5..bd9f4b6f80 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -133,3 +133,4 @@ Currently experimental features are: - Ingester: Active Series Tracker - Per-tenant `active_series_trackers` configuration in runtime config overrides - Counts active series matching PromQL label matchers and exposes `cortex_ingester_active_series_per_tracker` metric + - `cortex_ingester_active_metric_names` metric exposing the number of unique metric names per user in the ingester head diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 6116a31899..c68c6719de 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1152,6 +1152,7 @@ func (i *Ingester) updateActiveSeries(ctx context.Context) { userDB.activeSeries.Purge(purgeTime) i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.Active())) i.metrics.activeNHSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.ActiveNativeHistogram())) + i.metrics.activeMetricNamesPerUser.WithLabelValues(userID).Set(float64(userDB.seriesInMetric.ActiveMetricNames())) if err := userDB.labelSetCounter.UpdateMetric(ctx, userDB, i.metrics); err != nil { level.Warn(i.logger).Log("msg", "failed to update per labelSet metrics", "user", userID, "err", err) } @@ -3041,6 +3042,7 @@ func (i *Ingester) closeAllTSDB() { i.metrics.memUsers.Dec() i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) i.metrics.activeNHSeriesPerUser.DeleteLabelValues(userID) + i.metrics.activeMetricNamesPerUser.DeleteLabelValues(userID) }(userDB) } diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 238c578e65..437ea26ab3 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -57,6 +57,7 @@ type ingesterMetrics struct { activeSeriesPerUser *prometheus.GaugeVec activeNHSeriesPerUser *prometheus.GaugeVec + activeMetricNamesPerUser *prometheus.GaugeVec activeQueriedSeriesPerUser *prometheus.GaugeVec limitsPerLabelSet *prometheus.GaugeVec usagePerLabelSet *prometheus.GaugeVec @@ -298,6 +299,12 @@ func newIngesterMetrics(r prometheus.Registerer, Help: "Number of currently active native histogram series per user.", }, []string{"user"}), + // Not registered automatically, but only if activeSeriesEnabled is true. + activeMetricNamesPerUser: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_ingester_active_metric_names", + Help: "Number of unique metric names in the ingester head per user.", + }, []string{"user"}), + // Not registered automatically, but only if activeSeriesEnabled is true. activeSeriesPerTracker: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_ingester_active_series_per_tracker", @@ -349,6 +356,7 @@ func newIngesterMetrics(r prometheus.Registerer, if activeSeriesEnabled && r != nil { r.MustRegister(m.activeSeriesPerUser) r.MustRegister(m.activeNHSeriesPerUser) + r.MustRegister(m.activeMetricNamesPerUser) r.MustRegister(m.activeSeriesPerTracker) } @@ -380,6 +388,7 @@ func (m *ingesterMetrics) deletePerUserMetrics(userID string) { m.memMetadataRemovedTotal.DeleteLabelValues(userID) m.activeSeriesPerUser.DeleteLabelValues(userID) m.activeNHSeriesPerUser.DeleteLabelValues(userID) + m.activeMetricNamesPerUser.DeleteLabelValues(userID) m.activeSeriesPerTracker.DeletePartialMatch(prometheus.Labels{"user": userID}) m.activeQueriedSeriesPerUser.DeletePartialMatch(prometheus.Labels{"user": userID}) m.usagePerLabelSet.DeletePartialMatch(prometheus.Labels{"user": userID}) diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 2918c8993a..f7531d64cb 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -86,6 +86,17 @@ func (m *metricCounter) increaseSeriesForMetric(metric string) { shard.mtx.Unlock() } +// ActiveMetricNames returns the total number of unique metric names tracked across all shards. +func (m *metricCounter) ActiveMetricNames() int { + total := 0 + for i := range m.shards { + m.shards[i].mtx.Lock() + total += len(m.shards[i].m) + m.shards[i].mtx.Unlock() + } + return total +} + type labelSetCounterEntry struct { count int labels labels.Labels diff --git a/pkg/ingester/user_state_test.go b/pkg/ingester/user_state_test.go index 38be322854..bccc00e892 100644 --- a/pkg/ingester/user_state_test.go +++ b/pkg/ingester/user_state_test.go @@ -378,3 +378,36 @@ func (ir *mockIndexReader) LabelNamesFor(ctx context.Context, postings index.Pos } func (ir *mockIndexReader) Close() error { return nil } + +func TestMetricCounter_ActiveMetricNames(t *testing.T) { + limits := validation.Limits{MaxLocalSeriesPerMetric: 100} + overrides := validation.NewOverrides(limits, nil) + limiter := NewLimiter(overrides, nil, util.ShardingStrategyDefault, true, 3, false, "") + mc := newMetricCounter(limiter, nil) + + // Initially zero. + assert.Equal(t, 0, mc.ActiveMetricNames()) + + // Add series for 3 different metrics. + mc.increaseSeriesForMetric("metric_a") + mc.increaseSeriesForMetric("metric_a") + mc.increaseSeriesForMetric("metric_b") + mc.increaseSeriesForMetric("metric_c") + assert.Equal(t, 3, mc.ActiveMetricNames()) + + // Remove all series for metric_b. + mc.decreaseSeriesForMetric("metric_b") + assert.Equal(t, 2, mc.ActiveMetricNames()) + + // Remove one series for metric_a (still has one left). + mc.decreaseSeriesForMetric("metric_a") + assert.Equal(t, 2, mc.ActiveMetricNames()) + + // Remove last series for metric_a. + mc.decreaseSeriesForMetric("metric_a") + assert.Equal(t, 1, mc.ActiveMetricNames()) + + // Remove last series for metric_c. + mc.decreaseSeriesForMetric("metric_c") + assert.Equal(t, 0, mc.ActiveMetricNames()) +}