Emit dashboard chart data as Datadog metrics

src/commands/generate_charts.rs

@@ -28,7 +28,7 @@ const PALETTE: &[&str] = &[
 
 const EXCLUDE_LABELS: &[&str] = &["no-changelog", "meta: awaiting author"];
 
-fn slugify(s: &str) -> String {
+pub(crate) fn slugify(s: &str) -> String {
     s.chars()
         .map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' })
         .collect::<String>()
@@ -265,7 +265,7 @@ pub fn label_counts_over_time(rows: &[&HashMap<String, String>]) -> Chart {
     }
 
     let mut top_labels: Vec<(String, i64)> = label_totals.into_iter().collect();
-    top_labels.sort_by(|a, b| b.1.cmp(&a.1));
+    top_labels.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
     top_labels.truncate(8);
     let top_labels: Vec<String> = top_labels.into_iter().map(|(l, _)| l).collect();
 
@@ -376,7 +376,7 @@ pub fn integration_trends(
             (*col, total)
         })
         .collect();
-    totals.sort_by(|a, b| b.1.cmp(&a.1));
+    totals.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
     totals.truncate(5);
 
     let top_cols: Vec<&String> = totals.into_iter().map(|(c, _)| c).collect();
@@ -443,7 +443,7 @@ pub fn contributor_heatmap(rows: &[&HashMap<String, String>]) -> Option<Chart> {
         *user_totals.entry(user).or_insert(0) += count;
     }
     let mut top_users: Vec<(String, i64)> = user_totals.into_iter().collect();
-    top_users.sort_by(|a, b| b.1.cmp(&a.1));
+    top_users.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
     top_users.truncate(10);
     let top_users: Vec<String> = top_users.into_iter().map(|(u, _)| u).collect();
 
@@ -739,10 +739,10 @@ fn ai_stats_entry(stats: &AiStats) -> Result<ChartEntry> {
 
 // ── HTML generation ─────────────────────────────────────────────────────────────
 
-struct ChartEntry {
-    title: String,
+pub(crate) struct ChartEntry {
+    pub(crate) title: String,
     note: Option<String>,
-    json: String,
+    pub(crate) json: String,
     height_px: u32,
     extra_html: Option<String>,
 }
@@ -1046,6 +1046,61 @@ fn render_index_html(repos: &[(String, String)]) -> String {
 /// Generate charts for a single repo.
 /// `repo` is in the form "owner/name" (e.g. "vectordotdev/vector").
 pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -> Result<()> {
+    let (owner, name) = repo.split_once('/')
+        .with_context(|| format!("repo must be owner/name, got '{repo}'"))?;
+    let repo_display = format!("{owner}/{name}");
+    let Built { sections, yearly_stats } = build(input_dir, repo, start)?;
+    if let Some(yearly_stats) = &yearly_stats {
+        update_yearly_contributors_md(name, yearly_stats)?;
+    }
+
+    // ── Write output ──
+    let repo_out_dir = format!("{output_dir}/{name}");
+    fs::create_dir_all(&repo_out_dir)
+        .with_context(|| format!("creating directory {repo_out_dir}"))?;
+
+    let data_notes = read_data_notes(name);
+    let github_url = format!("https://github.com/{owner}/{name}");
+    let html = render_html(&repo_display, &sections, data_notes.as_deref(), &github_url);
+    let out_path = format!("{repo_out_dir}/index.html");
+    fs::write(&out_path, &html)
+        .with_context(|| format!("writing {out_path}"))?;
+
+    println!("Generated: {out_path}");
+
+    // Write .repo marker so the index knows the full owner/name
+    let repo_marker = Path::new(output_dir).join(name).join(".repo");
+    fs::write(&repo_marker, format!("{owner}/{name}"))?;
+
+    // Regenerate the top-level index by scanning for subdirectories with .repo
+    let mut repos_found: Vec<(String, String)> = Vec::new();
+    if let Ok(entries) = fs::read_dir(output_dir) {
+        let mut sorted: Vec<_> = entries.filter_map(|e| e.ok()).collect();
+        sorted.sort_by_key(|e| e.file_name());
+        for entry in sorted {
+            let path = entry.path();
+            let repo_file = path.join(".repo");
+            if path.is_dir() && repo_file.exists()
+                && let Ok(content) = fs::read_to_string(&repo_file)
+                    && let Some((o, n)) = content.trim().split_once('/') {
+                        repos_found.push((o.to_string(), n.to_string()));
+                    }
+        }
+    }
+    generate_index(output_dir, &repos_found)?;
+
+    Ok(())
+}
+
+/// Chart sections plus yearly contributor stats, without writing anything.
+pub(crate) struct Built {
+    pub(crate) sections: Vec<(&'static str, Vec<ChartEntry>)>,
+    pub(crate) yearly_stats: Option<BTreeMap<String, (i64, i64)>>,
+}
+
+/// Builds every chart for one repo from summary CSVs. Shared by the HTML
+/// dashboards and the Datadog chart metrics so both stay identical.
+pub(crate) fn build(input_dir: &str, repo: &str, start: Option<&str>) -> Result<Built> {
     use chrono::{Datelike, Utc};
     let two_years_ago = {
         let now = Utc::now();
@@ -1057,7 +1112,6 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -
     let (owner, name) = repo.split_once('/')
         .with_context(|| format!("repo must be owner/name, got '{repo}'"))?;
     let prefix = format!("{owner}_{name}");
-    let repo_display = format!("{owner}/{name}");
 
     // ── Load CSVs ──
     let issues_monthly = read_csv(&format!("{input_dir}/{prefix}_issues.monthly_summary.csv"))?;
@@ -1197,6 +1251,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -
     }
 
     // Contributors section
+    let mut yearly = None;
     if !pr_contributor.is_empty() {
         let mut entries = Vec::new();
         let filtered_contrib = filter_by_start(&pr_contributor, start);
@@ -1267,9 +1322,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -
         if !entries.is_empty() {
             sections.push(("Contributors", entries));
         }
-        if !pr_contributor.is_empty() {
-            update_yearly_contributors_md(name, &yearly_stats)?;
-        }
+        yearly = Some(yearly_stats);
     }
 
     // AI Code Review section
@@ -1278,42 +1331,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -
         sections.push(("AI Code Review", vec![entry]));
     }
 
-    // ── Write output ──
-    let repo_out_dir = format!("{output_dir}/{name}");
-    fs::create_dir_all(&repo_out_dir)
-        .with_context(|| format!("creating directory {repo_out_dir}"))?;
-
-    let data_notes = read_data_notes(name);
-    let github_url = format!("https://github.com/{owner}/{name}");
-    let html = render_html(&repo_display, &sections, data_notes.as_deref(), &github_url);
-    let out_path = format!("{repo_out_dir}/index.html");
-    fs::write(&out_path, &html)
-        .with_context(|| format!("writing {out_path}"))?;
-
-    println!("Generated: {out_path}");
-
-    // Write .repo marker so the index knows the full owner/name
-    let repo_marker = Path::new(output_dir).join(name).join(".repo");
-    fs::write(&repo_marker, format!("{owner}/{name}"))?;
-
-    // Regenerate the top-level index by scanning for subdirectories with .repo
-    let mut repos_found: Vec<(String, String)> = Vec::new();
-    if let Ok(entries) = fs::read_dir(output_dir) {
-        let mut sorted: Vec<_> = entries.filter_map(|e| e.ok()).collect();
-        sorted.sort_by_key(|e| e.file_name());
-        for entry in sorted {
-            let path = entry.path();
-            let repo_file = path.join(".repo");
-            if path.is_dir() && repo_file.exists()
-                && let Ok(content) = fs::read_to_string(&repo_file)
-                    && let Some((o, n)) = content.trim().split_once('/') {
-                        repos_found.push((o.to_string(), n.to_string()));
-                    }
-        }
-    }
-    generate_index(output_dir, &repos_found)?;
-
-    Ok(())
+    Ok(Built { sections, yearly_stats: yearly })
 }
 
 /// Generate the overview index.html listing all repos.

src/commands/push_metrics.rs

@@ -1,4 +1,5 @@
 use crate::commands::fetch_issues::parse_since;
+use crate::commands::{generate_charts, generate_summaries, workflows};
 use crate::config::Config;
 use anyhow::{Context, Result};
 use chrono::{DateTime, SecondsFormat, Utc};
@@ -114,7 +115,8 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> {
             },
         )?);
     }
-    let series = coalesce_series(historical_series);
+    let mut series = coalesce_series(historical_series);
+    series.extend(chart_metrics(&db_path, config, prefix, now)?);
 
     if output_json {
         println!("{}", serde_json::to_string(&json_batch_output(&series)?)?);
@@ -179,6 +181,119 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> {
     Ok(())
 }
 
+/// Emits every chart on the published dashboards as `{prefix}.chart` gauges,
+/// built by the same code that renders the HTML so the numbers match exactly.
+/// All points share one timestamp; the `snapshot` tag lets dashboards keep
+/// only the latest run and ignore labels that have since dropped out.
+fn chart_metrics(db_path: &str, config: &Config, prefix: &str, now: i64) -> Result<Vec<MetricSeries>> {
+    generate_summaries::run(db_path, config)?;
+    let repo = format!("{}/{}", config.org, config.repo);
+    let start = workflows::default_chart_start();
+    let built = generate_charts::build("out/summaries", &repo, Some(&start))?;
+
+    let base = vec![
+        format!("repo:{}", sanitize_tag_value(&repo)),
+        format!("snapshot:{now}"),
+    ];
+    let metric = format!("{prefix}.chart");
+    let mut series = Vec::new();
+    for entry in built.sections.iter().flat_map(|(_, entries)| entries) {
+        let chart: serde_json::Value = serde_json::from_str(&entry.json)?;
+        let chart_tag = format!("chart:{}", generate_charts::slugify(&entry.title));
+        for point in chart_points(&chart) {
+            let mut tags = base.clone();
+            tags.push(chart_tag.clone());
+            tags.extend(point.tags);
+            series.push(MetricSeries::gauge(metric.clone(), point.value, now, tags));
+        }
+    }
+    if let Some(yearly) = built.yearly_stats {
+        for (year, (new, returning)) in yearly {
+            for (name, value) in [("unique", new + returning), ("new", new), ("returning", returning)] {
+                let mut tags = base.clone();
+                tags.extend([
+                    "chart:contributors-yearly-table".to_string(),
+                    format!("series:{name}"),
+                    format!("x:{year}"),
+                ]);
+                series.push(MetricSeries::gauge(metric.clone(), value, now, tags));
+            }
+        }
+    }
+    Ok(series)
+}
+
+struct ChartPoint {
+    value: i64,
+    tags: Vec<String>,
+}
+
+/// Flattens an ECharts option (line, bar, horizontal bar, heatmap) into points.
+fn chart_points(chart: &serde_json::Value) -> Vec<ChartPoint> {
+    let categories = |axis: &str| -> Vec<String> {
+        chart[axis][0]["data"]
+            .as_array()
+            .map(|values| values.iter().map(|v| v.as_str().unwrap_or_default().to_string()).collect())
+            .unwrap_or_default()
+    };
+    let x_categories = categories("xAxis");
+    let y_categories = categories("yAxis");
+    // Horizontal bars put the categories on the y axis.
+    let categories = if x_categories.is_empty() { &y_categories } else { &x_categories };
+
+    let mut points = Vec::new();
+    for (order, series) in chart["series"].as_array().into_iter().flatten().enumerate() {
+        let name = series["name"].as_str().unwrap_or_default();
+        let series_color = series["itemStyle"]["color"].as_str();
+        for (index, datum) in series["data"].as_array().into_iter().flatten().enumerate() {
+            let mut tags = vec![format!("series:{}", chart_tag(name)), format!("order:{order}")];
+            let (value, color) = if let Some(cell) = datum.as_array() {
+                // Heatmap cell: [x index, y index, value].
+                let x = cell[0].as_u64().unwrap_or_default() as usize;
+                let y = cell[1].as_u64().unwrap_or_default() as usize;
+                tags.extend([
+                    format!("x:{}", chart_tag(&x_categories[x])),
+                    format!("x_order:{x}"),
+                    format!("y:{}", chart_tag(&y_categories[y])),
+                    format!("y_order:{y}"),
+                ]);
+                (cell[2].as_i64(), None)
+            } else {
+                tags.extend([
+                    format!("x:{}", chart_tag(&categories[index])),
+                    format!("x_order:{index}"),
+                ]);
+                match datum.get("value") {
+                    Some(value) => (value.as_i64(), datum["itemStyle"]["color"].as_str()),
+                    None => (datum.as_i64(), None),
+                }
+            };
+            if let Some(color) = color.or(series_color) {
+                tags.push(format!("color:{}", color.trim_start_matches('#').to_lowercase()));
+            }
+            points.push(ChartPoint { value: value.unwrap_or_default(), tags });
+        }
+    }
+    points
+}
+
+/// Tag-safe label encoding that dashboards can reverse: spaces become `.`
+/// (no GitHub label or login here contains a dot). Datadog lowercases tag
+/// values, so original casing is lost.
+fn chart_tag(value: &str) -> String {
+    value
+        .trim()
+        .to_lowercase()
+        .chars()
+        .map(|c| match c {
+            ' ' => '.',
+            c if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '/') => c,
+            _ => '_',
+        })
+        .take(200)
+        .collect()
+}
+
 fn collect_metrics(
     conn: &Connection,
     config: &Config,
@@ -847,6 +962,44 @@ mod tests {
         );
     }
 
+    #[test]
+    fn flattens_line_bar_and_heatmap_charts() {
+        let line = serde_json::json!({
+            "xAxis": [{"data": ["2026-07", "2026-08"]}],
+            "series": [{"name": "type: bug", "itemStyle": {"color": "#FF4C4C"}, "data": [3, 4]}]
+        });
+        let points = chart_points(&line);
+        assert_eq!(points.len(), 2);
+        assert_eq!(points[1].value, 4);
+        assert_eq!(
+            points[1].tags,
+            ["series:type:.bug", "order:0", "x:2026-08", "x_order:1", "color:ff4c4c"]
+        );
+
+        let horizontal = serde_json::json!({
+            "xAxis": [{"type": "value"}],
+            "yAxis": [{"data": ["domain: external docs"]}],
+            "series": [{"name": "Count", "data": [{"value": 7, "itemStyle": {"color": "#afab7e"}}]}]
+        });
+        let points = chart_points(&horizontal);
+        assert_eq!(points[0].value, 7);
+        assert!(points[0].tags.contains(&"x:domain:.external.docs".to_string()));
+        assert!(points[0].tags.contains(&"color:afab7e".to_string()));
+
+        let heatmap = serde_json::json!({
+            "xAxis": [{"data": ["2026-07", "2026-08"]}],
+            "yAxis": [{"data": ["alice", "Bob"]}],
+            "series": [{"name": "PRs", "data": [[1, 1, 5]]}]
+        });
+        let points = chart_points(&heatmap);
+        assert_eq!(points[0].value, 5);
+        assert_eq!(
+            points[0].tags,
+            ["series:prs", "order:0", "x:2026-08", "x_order:1", "y:bob", "y_order:1"]
+        );
+        assert_eq!(chart_tag("dependabot[bot]"), "dependabot_bot_");
+    }
+
     #[test]
     fn validates_datadog_site_and_metric_prefix() {
         assert_eq!(

src/commands/workflows.rs

@@ -76,6 +76,13 @@ fn build_repo_db(config: &Config) -> Result<String> {
     Ok(db)
 }
 
+/// First month shown on the published dashboards (last 12 complete months).
+pub fn default_chart_start() -> String {
+    let now = Utc::now();
+    let total_months = now.year() * 12 + now.month() as i32 - 13;
+    format!("{}-{:02}", total_months / 12, total_months % 12 + 1)
+}
+
 /// Builds DB + summaries + HTML charts for a single repo.
 pub fn generate_all(repo_str: &str, start: Option<&str>) -> Result<()> {
     let repo = Repo::parse(repo_str)?;
@@ -84,11 +91,7 @@ pub fn generate_all(repo_str: &str, start: Option<&str>) -> Result<()> {
 
     generate_summaries::run(&db, &config)?;
 
-    let default_start = {
-        let now = Utc::now();
-        let total_months = now.year() * 12 + now.month() as i32 - 13;
-        format!("{}-{:02}", total_months / 12, total_months % 12 + 1)
-    };
+    let default_start = default_chart_start();
     let start_arg = start.unwrap_or(&default_start);
     generate_charts::run("out/summaries", repo_str, "docs", Some(start_arg))?;
     Ok(())