Ensure ceems_tool accounts multiple instances of emission collectors

.github/workflows/step_docs.yml

@@ -54,7 +54,8 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     needs: build-docs
-    if: github.ref == 'refs/heads/main' || github.ref_type == 'tag'
+    # Update docs only for tagged versions
+    if: github.ref_type == 'tag'
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

cmd/ceems_tool/rules.go

@@ -41,6 +41,10 @@ const (
 	amdDevExporterPkgPowerMetric = "gpu_package_power"
 )
 
+const (
+	emissionsMetric = "ceems_emissions_gCo2_kWh"
+)
+
 var (
 	seriesNames = []string{
 		"ceems_compute_unit_cpu_user_seconds_total",
@@ -51,7 +55,7 @@ var (
 		redfishPowerMetric,
 		crayPowerMetric,
 		hwmonPowerMetric,
-		"ceems_emissions_gCo2_kWh",
+		emissionsMetric,
 		dcgmPowerMetric,
 		dcgmInstantPowerMetric,
 		amdSMIPowerMetric,
@@ -129,6 +133,7 @@ type rulesTemplateData struct {
 	EmissionFactor     EmissionFactor
 	Providers          model.LabelValues
 	CountryCode        string
+	Instance           string
 	RateInterval       string
 	EvaluationInterval string
 }
@@ -229,6 +234,8 @@ func CreatePromRecordingRules(
 
 	var providers model.LabelValues
 
+	var emissionCollectorInstance string
+
 	if emissionFactorValue == 0 {
 		// If no emission factor value has been passed, attempt to get from time series or
 		// static OWID data
@@ -244,6 +251,9 @@ func CreatePromRecordingRules(
 				}
 			}
 		}
+
+		// Get instance of emission collector
+		emissionCollectorInstance = efInstance(ctx, api, stime, etime)
 	} else {
 		emissionFactor = EmissionFactor{Provider: "custom", Value: emissionFactorValue}
 	}
@@ -387,6 +397,7 @@ func CreatePromRecordingRules(
 			EmissionFactor:     emissionFactor,
 			Providers:          providers,
 			CountryCode:        countryCode,
+			Instance:           emissionCollectorInstance,
 			RateInterval:       rateInterval.String(),
 			EvaluationInterval: evalInterval.String(),
 		}
@@ -428,10 +439,36 @@ func scrapeIntervals(ctx context.Context, api v1.API) (map[string]time.Duration,
 	return scrapeIntervals, nil
 }
 
+// efInstance returns first found instance of emission factor metric. In case when operators run emissions collector on
+// multiple instances, it comes handy.
+func efInstance(ctx context.Context, api v1.API, start time.Time, end time.Time) string {
+	// First fetch all the available series of emissions
+	series, _, err := api.Series(ctx, []string{emissionsMetric}, start, end)
+	if err != nil || len(series) == 0 {
+		return ""
+	}
+
+	var instances []string
+	for _, s := range series {
+		instances = append(instances, string(s["instance"]))
+	}
+
+	// Remove duplicates and if more than one instance is found, it means multiple
+	// instances of emissions collectors are running which is not ideal. Emit a warning
+	instances = slices.Compact(instances)
+	slices.Sort(instances)
+
+	if len(instances) > 1 {
+		fmt.Fprintln(os.Stderr, `WARNING: multiple instances of emissions collectors are detected. The generated recording rules might not work in this case. Ensure there is only one single instance of emissions collector running in the cluster.`)
+	}
+
+	return instances[0]
+}
+
 // efProviders returns a slice of available emission factor providers.
 func efProviders(ctx context.Context, api v1.API, start time.Time, end time.Time, countryCode string, disableProviders bool) (model.LabelValues, error) {
 	// Run query to get label values.
-	matcher := fmt.Sprintf(`ceems_emissions_gCo2_kWh{country_code="%s"}`, countryCode)
+	matcher := fmt.Sprintf(`%s{country_code="%s"}`, emissionsMetric, countryCode)
 
 	providers, _, err := api.LabelValues(ctx, "provider", []string{matcher}, start, end) // Ignoring warnings for now.
 	if err != nil {

cmd/ceems_tool/rules/cpu-cray.rules

@@ -108,7 +108,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
-            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"} 
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -192,7 +196,11 @@ groups:
             (
                 job:ceems_host_power_watts:pue{job="{{$.Job}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.Job}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.Job}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}

cmd/ceems_tool/rules/cpu-ipmi-redfish-hwmon.rules

@@ -142,7 +142,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
-            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"} 
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -218,7 +222,11 @@ groups:
             (
                 job:ceems_host_power_watts:pue{job="{{$.Job}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.Job}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.Job}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}

cmd/ceems_tool/rules/cpu-rapl.rules

@@ -95,7 +95,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
-            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"} 
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -179,7 +183,11 @@ groups:
             (
                 job:ceems_host_power_watts:pue{job="{{$.Job}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.Job}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.Job}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}

cmd/ceems_tool/rules/gpu-amd-device-metrics.rules

@@ -95,7 +95,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
-            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"} 
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -178,7 +182,11 @@ groups:
             (
                 job:ceems_gpu_power_watts:pue{job="{{$.GPUJob}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}

cmd/ceems_tool/rules/gpu-amd-smi.rules

@@ -63,7 +63,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
             ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -116,7 +120,11 @@ groups:
             (
                 job:ceems_gpu_power_watts:pue{job="{{$.GPUJob}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}

cmd/ceems_tool/rules/gpu-nvidia.rules

@@ -110,7 +110,11 @@ groups:
               "(.*)"
             )
           * on (provider) group_left ()
+{{- if ne $.Instance "" }}
+            ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}",instance="{{$.Instance}}"}
+{{- else }}
             ceems_emissions_gCo2_kWh{country_code="{{$.CountryCode}}",provider="{{$v}}"}
+{{- end }}
 {{ end }}
 {{- else }}
 {{- if gt $.EmissionFactor.Value 0.0 }}
@@ -189,7 +193,11 @@ groups:
             (
                 job:ceems_gpu_power_watts:pue{job="{{$.GPUJob}}"} / 3.6e+06
               * on (job) group_right ()
+{{- if ne $.Instance "" }}
+                label_replace(ceems_emissions_gCo2_kWh{instance="{{$.Instance}}"}, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- else }}
                 label_replace(ceems_emissions_gCo2_kWh, "job", "{{$.GPUJob}}", "instance", "(.*)")
+{{- end }}
             )
           )
 {{- else }}