From 9c924e099d3a6d0d8a7b43cd09ad49a0f97d0a9c Mon Sep 17 00:00:00 2001 From: "david.yaffe" Date: Mon, 28 Sep 2026 20:03:33 +0000 Subject: [PATCH] Emit dashboard chart data as Datadog metrics Sync-metrics and push-metrics now submit github.health.v2.chart gauges built from the same charts as the published Pages dashboards. Make tied top-N chart ordering deterministic. --- src/commands/generate_charts.rs | 112 +++++++++++++---------- src/commands/push_metrics.rs | 155 +++++++++++++++++++++++++++++++- src/commands/workflows.rs | 13 +-- 3 files changed, 227 insertions(+), 53 deletions(-) diff --git a/src/commands/generate_charts.rs b/src/commands/generate_charts.rs index db49ea1..8bbb584 100644 --- a/src/commands/generate_charts.rs +++ b/src/commands/generate_charts.rs @@ -28,7 +28,7 @@ const PALETTE: &[&str] = &[ const EXCLUDE_LABELS: &[&str] = &["no-changelog", "meta: awaiting author"]; -fn slugify(s: &str) -> String { +pub(crate) fn slugify(s: &str) -> String { s.chars() .map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' }) .collect::() @@ -265,7 +265,7 @@ pub fn label_counts_over_time(rows: &[&HashMap]) -> Chart { } let mut top_labels: Vec<(String, i64)> = label_totals.into_iter().collect(); - top_labels.sort_by(|a, b| b.1.cmp(&a.1)); + top_labels.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); top_labels.truncate(8); let top_labels: Vec = top_labels.into_iter().map(|(l, _)| l).collect(); @@ -376,7 +376,7 @@ pub fn integration_trends( (*col, total) }) .collect(); - totals.sort_by(|a, b| b.1.cmp(&a.1)); + totals.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); totals.truncate(5); let top_cols: Vec<&String> = totals.into_iter().map(|(c, _)| c).collect(); @@ -443,7 +443,7 @@ pub fn contributor_heatmap(rows: &[&HashMap]) -> Option { *user_totals.entry(user).or_insert(0) += count; } let mut top_users: Vec<(String, i64)> = user_totals.into_iter().collect(); - top_users.sort_by(|a, b| b.1.cmp(&a.1)); + top_users.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); top_users.truncate(10); let top_users: Vec = top_users.into_iter().map(|(u, _)| u).collect(); @@ -739,10 +739,10 @@ fn ai_stats_entry(stats: &AiStats) -> Result { // ── HTML generation ───────────────────────────────────────────────────────────── -struct ChartEntry { - title: String, +pub(crate) struct ChartEntry { + pub(crate) title: String, note: Option, - json: String, + pub(crate) json: String, height_px: u32, extra_html: Option, } @@ -1046,6 +1046,61 @@ fn render_index_html(repos: &[(String, String)]) -> String { /// Generate charts for a single repo. /// `repo` is in the form "owner/name" (e.g. "vectordotdev/vector"). pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) -> Result<()> { + let (owner, name) = repo.split_once('/') + .with_context(|| format!("repo must be owner/name, got '{repo}'"))?; + let repo_display = format!("{owner}/{name}"); + let Built { sections, yearly_stats } = build(input_dir, repo, start)?; + if let Some(yearly_stats) = &yearly_stats { + update_yearly_contributors_md(name, yearly_stats)?; + } + + // ── Write output ── + let repo_out_dir = format!("{output_dir}/{name}"); + fs::create_dir_all(&repo_out_dir) + .with_context(|| format!("creating directory {repo_out_dir}"))?; + + let data_notes = read_data_notes(name); + let github_url = format!("https://github.com/{owner}/{name}"); + let html = render_html(&repo_display, §ions, data_notes.as_deref(), &github_url); + let out_path = format!("{repo_out_dir}/index.html"); + fs::write(&out_path, &html) + .with_context(|| format!("writing {out_path}"))?; + + println!("Generated: {out_path}"); + + // Write .repo marker so the index knows the full owner/name + let repo_marker = Path::new(output_dir).join(name).join(".repo"); + fs::write(&repo_marker, format!("{owner}/{name}"))?; + + // Regenerate the top-level index by scanning for subdirectories with .repo + let mut repos_found: Vec<(String, String)> = Vec::new(); + if let Ok(entries) = fs::read_dir(output_dir) { + let mut sorted: Vec<_> = entries.filter_map(|e| e.ok()).collect(); + sorted.sort_by_key(|e| e.file_name()); + for entry in sorted { + let path = entry.path(); + let repo_file = path.join(".repo"); + if path.is_dir() && repo_file.exists() + && let Ok(content) = fs::read_to_string(&repo_file) + && let Some((o, n)) = content.trim().split_once('/') { + repos_found.push((o.to_string(), n.to_string())); + } + } + } + generate_index(output_dir, &repos_found)?; + + Ok(()) +} + +/// Chart sections plus yearly contributor stats, without writing anything. +pub(crate) struct Built { + pub(crate) sections: Vec<(&'static str, Vec)>, + pub(crate) yearly_stats: Option>, +} + +/// Builds every chart for one repo from summary CSVs. Shared by the HTML +/// dashboards and the Datadog chart metrics so both stay identical. +pub(crate) fn build(input_dir: &str, repo: &str, start: Option<&str>) -> Result { use chrono::{Datelike, Utc}; let two_years_ago = { let now = Utc::now(); @@ -1057,7 +1112,6 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) - let (owner, name) = repo.split_once('/') .with_context(|| format!("repo must be owner/name, got '{repo}'"))?; let prefix = format!("{owner}_{name}"); - let repo_display = format!("{owner}/{name}"); // ── Load CSVs ── let issues_monthly = read_csv(&format!("{input_dir}/{prefix}_issues.monthly_summary.csv"))?; @@ -1197,6 +1251,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) - } // Contributors section + let mut yearly = None; if !pr_contributor.is_empty() { let mut entries = Vec::new(); let filtered_contrib = filter_by_start(&pr_contributor, start); @@ -1267,9 +1322,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) - if !entries.is_empty() { sections.push(("Contributors", entries)); } - if !pr_contributor.is_empty() { - update_yearly_contributors_md(name, &yearly_stats)?; - } + yearly = Some(yearly_stats); } // AI Code Review section @@ -1278,42 +1331,7 @@ pub fn run(input_dir: &str, repo: &str, output_dir: &str, start: Option<&str>) - sections.push(("AI Code Review", vec![entry])); } - // ── Write output ── - let repo_out_dir = format!("{output_dir}/{name}"); - fs::create_dir_all(&repo_out_dir) - .with_context(|| format!("creating directory {repo_out_dir}"))?; - - let data_notes = read_data_notes(name); - let github_url = format!("https://github.com/{owner}/{name}"); - let html = render_html(&repo_display, §ions, data_notes.as_deref(), &github_url); - let out_path = format!("{repo_out_dir}/index.html"); - fs::write(&out_path, &html) - .with_context(|| format!("writing {out_path}"))?; - - println!("Generated: {out_path}"); - - // Write .repo marker so the index knows the full owner/name - let repo_marker = Path::new(output_dir).join(name).join(".repo"); - fs::write(&repo_marker, format!("{owner}/{name}"))?; - - // Regenerate the top-level index by scanning for subdirectories with .repo - let mut repos_found: Vec<(String, String)> = Vec::new(); - if let Ok(entries) = fs::read_dir(output_dir) { - let mut sorted: Vec<_> = entries.filter_map(|e| e.ok()).collect(); - sorted.sort_by_key(|e| e.file_name()); - for entry in sorted { - let path = entry.path(); - let repo_file = path.join(".repo"); - if path.is_dir() && repo_file.exists() - && let Ok(content) = fs::read_to_string(&repo_file) - && let Some((o, n)) = content.trim().split_once('/') { - repos_found.push((o.to_string(), n.to_string())); - } - } - } - generate_index(output_dir, &repos_found)?; - - Ok(()) + Ok(Built { sections, yearly_stats: yearly }) } /// Generate the overview index.html listing all repos. diff --git a/src/commands/push_metrics.rs b/src/commands/push_metrics.rs index 7ce80c9..af6a8da 100644 --- a/src/commands/push_metrics.rs +++ b/src/commands/push_metrics.rs @@ -1,4 +1,5 @@ use crate::commands::fetch_issues::parse_since; +use crate::commands::{generate_charts, generate_summaries, workflows}; use crate::config::Config; use anyhow::{Context, Result}; use chrono::{DateTime, SecondsFormat, Utc}; @@ -114,7 +115,8 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> { }, )?); } - let series = coalesce_series(historical_series); + let mut series = coalesce_series(historical_series); + series.extend(chart_metrics(&db_path, config, prefix, now)?); if output_json { println!("{}", serde_json::to_string(&json_batch_output(&series)?)?); @@ -179,6 +181,119 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> { Ok(()) } +/// Emits every chart on the published dashboards as `{prefix}.chart` gauges, +/// built by the same code that renders the HTML so the numbers match exactly. +/// All points share one timestamp; the `snapshot` tag lets dashboards keep +/// only the latest run and ignore labels that have since dropped out. +fn chart_metrics(db_path: &str, config: &Config, prefix: &str, now: i64) -> Result> { + generate_summaries::run(db_path, config)?; + let repo = format!("{}/{}", config.org, config.repo); + let start = workflows::default_chart_start(); + let built = generate_charts::build("out/summaries", &repo, Some(&start))?; + + let base = vec![ + format!("repo:{}", sanitize_tag_value(&repo)), + format!("snapshot:{now}"), + ]; + let metric = format!("{prefix}.chart"); + let mut series = Vec::new(); + for entry in built.sections.iter().flat_map(|(_, entries)| entries) { + let chart: serde_json::Value = serde_json::from_str(&entry.json)?; + let chart_tag = format!("chart:{}", generate_charts::slugify(&entry.title)); + for point in chart_points(&chart) { + let mut tags = base.clone(); + tags.push(chart_tag.clone()); + tags.extend(point.tags); + series.push(MetricSeries::gauge(metric.clone(), point.value, now, tags)); + } + } + if let Some(yearly) = built.yearly_stats { + for (year, (new, returning)) in yearly { + for (name, value) in [("unique", new + returning), ("new", new), ("returning", returning)] { + let mut tags = base.clone(); + tags.extend([ + "chart:contributors-yearly-table".to_string(), + format!("series:{name}"), + format!("x:{year}"), + ]); + series.push(MetricSeries::gauge(metric.clone(), value, now, tags)); + } + } + } + Ok(series) +} + +struct ChartPoint { + value: i64, + tags: Vec, +} + +/// Flattens an ECharts option (line, bar, horizontal bar, heatmap) into points. +fn chart_points(chart: &serde_json::Value) -> Vec { + let categories = |axis: &str| -> Vec { + chart[axis][0]["data"] + .as_array() + .map(|values| values.iter().map(|v| v.as_str().unwrap_or_default().to_string()).collect()) + .unwrap_or_default() + }; + let x_categories = categories("xAxis"); + let y_categories = categories("yAxis"); + // Horizontal bars put the categories on the y axis. + let categories = if x_categories.is_empty() { &y_categories } else { &x_categories }; + + let mut points = Vec::new(); + for (order, series) in chart["series"].as_array().into_iter().flatten().enumerate() { + let name = series["name"].as_str().unwrap_or_default(); + let series_color = series["itemStyle"]["color"].as_str(); + for (index, datum) in series["data"].as_array().into_iter().flatten().enumerate() { + let mut tags = vec![format!("series:{}", chart_tag(name)), format!("order:{order}")]; + let (value, color) = if let Some(cell) = datum.as_array() { + // Heatmap cell: [x index, y index, value]. + let x = cell[0].as_u64().unwrap_or_default() as usize; + let y = cell[1].as_u64().unwrap_or_default() as usize; + tags.extend([ + format!("x:{}", chart_tag(&x_categories[x])), + format!("x_order:{x}"), + format!("y:{}", chart_tag(&y_categories[y])), + format!("y_order:{y}"), + ]); + (cell[2].as_i64(), None) + } else { + tags.extend([ + format!("x:{}", chart_tag(&categories[index])), + format!("x_order:{index}"), + ]); + match datum.get("value") { + Some(value) => (value.as_i64(), datum["itemStyle"]["color"].as_str()), + None => (datum.as_i64(), None), + } + }; + if let Some(color) = color.or(series_color) { + tags.push(format!("color:{}", color.trim_start_matches('#').to_lowercase())); + } + points.push(ChartPoint { value: value.unwrap_or_default(), tags }); + } + } + points +} + +/// Tag-safe label encoding that dashboards can reverse: spaces become `.` +/// (no GitHub label or login here contains a dot). Datadog lowercases tag +/// values, so original casing is lost. +fn chart_tag(value: &str) -> String { + value + .trim() + .to_lowercase() + .chars() + .map(|c| match c { + ' ' => '.', + c if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '/') => c, + _ => '_', + }) + .take(200) + .collect() +} + fn collect_metrics( conn: &Connection, config: &Config, @@ -847,6 +962,44 @@ mod tests { ); } + #[test] + fn flattens_line_bar_and_heatmap_charts() { + let line = serde_json::json!({ + "xAxis": [{"data": ["2026-07", "2026-08"]}], + "series": [{"name": "type: bug", "itemStyle": {"color": "#FF4C4C"}, "data": [3, 4]}] + }); + let points = chart_points(&line); + assert_eq!(points.len(), 2); + assert_eq!(points[1].value, 4); + assert_eq!( + points[1].tags, + ["series:type:.bug", "order:0", "x:2026-08", "x_order:1", "color:ff4c4c"] + ); + + let horizontal = serde_json::json!({ + "xAxis": [{"type": "value"}], + "yAxis": [{"data": ["domain: external docs"]}], + "series": [{"name": "Count", "data": [{"value": 7, "itemStyle": {"color": "#afab7e"}}]}] + }); + let points = chart_points(&horizontal); + assert_eq!(points[0].value, 7); + assert!(points[0].tags.contains(&"x:domain:.external.docs".to_string())); + assert!(points[0].tags.contains(&"color:afab7e".to_string())); + + let heatmap = serde_json::json!({ + "xAxis": [{"data": ["2026-07", "2026-08"]}], + "yAxis": [{"data": ["alice", "Bob"]}], + "series": [{"name": "PRs", "data": [[1, 1, 5]]}] + }); + let points = chart_points(&heatmap); + assert_eq!(points[0].value, 5); + assert_eq!( + points[0].tags, + ["series:prs", "order:0", "x:2026-08", "x_order:1", "y:bob", "y_order:1"] + ); + assert_eq!(chart_tag("dependabot[bot]"), "dependabot_bot_"); + } + #[test] fn validates_datadog_site_and_metric_prefix() { assert_eq!( diff --git a/src/commands/workflows.rs b/src/commands/workflows.rs index 0b54a69..9e0d4ed 100644 --- a/src/commands/workflows.rs +++ b/src/commands/workflows.rs @@ -76,6 +76,13 @@ fn build_repo_db(config: &Config) -> Result { Ok(db) } +/// First month shown on the published dashboards (last 12 complete months). +pub fn default_chart_start() -> String { + let now = Utc::now(); + let total_months = now.year() * 12 + now.month() as i32 - 13; + format!("{}-{:02}", total_months / 12, total_months % 12 + 1) +} + /// Builds DB + summaries + HTML charts for a single repo. pub fn generate_all(repo_str: &str, start: Option<&str>) -> Result<()> { let repo = Repo::parse(repo_str)?; @@ -84,11 +91,7 @@ pub fn generate_all(repo_str: &str, start: Option<&str>) -> Result<()> { generate_summaries::run(&db, &config)?; - let default_start = { - let now = Utc::now(); - let total_months = now.year() * 12 + now.month() as i32 - 13; - format!("{}-{:02}", total_months / 12, total_months % 12 + 1) - }; + let default_start = default_chart_start(); let start_arg = start.unwrap_or(&default_start); generate_charts::run("out/summaries", repo_str, "docs", Some(start_arg))?; Ok(())