From 75d7ed18307dc59c53528bb1b8c2b91eaf1b58a6 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 17 Jul 2026 13:00:43 +1000 Subject: [PATCH 1/4] Add the generated data-audit dashboard: scanner, tracker, 3-page site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/build_audit.py scans the 8 synced Python-family lecture repos on origin/main (clone + grep — never gh search code, which cannot find URLs), classifies every data reference into the hosting-pattern taxonomy, and renders a 3-page static dashboard (overview / migration tracker / full audit) via scripts/render_audit.py. Three sources of truth, reconciled on every build: - lectures/*.yml manifests — the migrated datasets - migration.yml (new) — the migration lifecycle + PR provenance, kept separate from the manifests so it can be archived when the migration programme completes without touching the permanent dataset records - scripts/audit_annotations.yml (new) — curated judgment (descriptions, provenance classes, live-API pedagogy) for not-yet-migrated references The strict scan fails on an unannotated data reference or a migration.yml status that disagrees with what the lectures actually read, so the dashboard cannot silently rot — the failure mode that killed the hand-built 2026-07-15 audit artifact within two days. Part of #20; taxonomy and design system carried over from that artifact. Co-Authored-By: Claude Fable 5 --- .gitignore | 5 + migration.yml | 105 ++++ scripts/audit_annotations.yml | 358 +++++++++++++ scripts/build_audit.py | 522 +++++++++++++++++++ scripts/render_audit.py | 919 ++++++++++++++++++++++++++++++++++ 5 files changed, 1909 insertions(+) create mode 100644 .gitignore create mode 100644 migration.yml create mode 100644 scripts/audit_annotations.yml create mode 100644 scripts/build_audit.py create mode 100644 scripts/render_audit.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95f2fb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# generated by scripts/build_audit.py — built fresh in CI, never committed +site/ +audit.json + +__pycache__/ diff --git a/migration.yml b/migration.yml new file mode 100644 index 0000000..61fcc0a --- /dev/null +++ b/migration.yml @@ -0,0 +1,105 @@ +# migration.yml — the migration lifecycle tracker for data-lectures. +# +# One record per dataset moving into this repo, holding the PROCESS facts the +# manifests deliberately do not: which PRs landed the file, which PRs repointed +# each consumer, and where the dataset sits in the lifecycle. Manifests +# (lectures/*.yml) are the permanent record of a dataset; this file documents +# a transition and can be archived once the migration programme completes, +# without touching a single manifest. +# +# The audit dashboard (scripts/build_audit.py) renders this file and VERIFIES +# it against reality on every build: a dataset marked `repointed` whose +# consumers still read an old URL (or vice versa) fails the consistency check. +# +# Lifecycle (status): +# pending identified for migration; nothing landed here yet +# landed file + manifest merged in data-lectures; consumers unchanged +# repointed every consumer reads this repo's interim raw URL +# final every consumer reads https://data.quantecon.org/lectures/… +# (gated on the Phase 4 DNS work — data-lectures#15) + +datasets: + lingcod_msy_recovery.csv: + pilot: P1 + status: repointed + prior_pattern: local-path + landed: + pr: QuantEcon/data-lectures#12 + date: 2026-07-15 + repoints: + - repo: QuantEcon/lecture-python-intro + pr: QuantEcon/lecture-python-intro#792 + date: 2026-07-17 + cutover: null + + realwage.csv: + pilot: P2 + status: repointed + prior_pattern: legacy + landed: + pr: QuantEcon/data-lectures#17 + date: 2026-07-17 + repoints: + - repo: QuantEcon/lecture-python-programming + pr: QuantEcon/lecture-python-programming#578 + date: 2026-07-17 + - repo: QuantEcon/lecture-python.myst + pr: QuantEcon/lecture-python.myst#973 + date: 2026-07-17 + cutover: null + + countries.csv: + pilot: P2 + status: repointed + prior_pattern: legacy + landed: + pr: QuantEcon/data-lectures#17 + date: 2026-07-17 + repoints: + - repo: QuantEcon/lecture-python-programming + pr: QuantEcon/lecture-python-programming#578 + date: 2026-07-17 + - repo: QuantEcon/lecture-python.myst + pr: QuantEcon/lecture-python.myst#973 + date: 2026-07-17 + cutover: null + + employ.csv: + pilot: P2 + status: repointed + prior_pattern: legacy # programming read the legacy repo; python.myst its own copy + landed: + pr: QuantEcon/data-lectures#17 + date: 2026-07-17 + repoints: + - repo: QuantEcon/lecture-python-programming + pr: QuantEcon/lecture-python-programming#578 + date: 2026-07-17 + - repo: QuantEcon/lecture-python.myst + pr: QuantEcon/lecture-python.myst#973 + date: 2026-07-17 + cutover: null + +# Planned waves that have not landed anything here yet. `datasets` names the +# files as the audit sees them today, so the dashboard can join the two views. +pending: + - pilot: P3 + scope: the LFS case — heavy_tails Forbes/cities set + SCF extracts (fold in high_dim_data) + datasets: + - forbes-global2000.csv + - forbes-billionaires.csv + - cities_us.csv + - cities_brazil.csv + - SCF_plus_mini.csv + - SCF_plus_mini_no_weights.csv + tracking: + - QuantEcon/data-lectures#2 + - QuantEcon/meta#337 + - pilot: P4 + scope: the dynamic-snapshot case — a maintained UNRATE twin for the incidental FRED consumers + datasets: [] # produces a NEW file here; no existing static file moves + tracking: + - QuantEcon/meta#338 + +# Interim → final URL cutover (status: repointed → final) is tracked as one +# sweep, not per dataset: QuantEcon/data-lectures#15, gated on Phase 4 DNS. diff --git a/scripts/audit_annotations.yml b/scripts/audit_annotations.yml new file mode 100644 index 0000000..c523de8 --- /dev/null +++ b/scripts/audit_annotations.yml @@ -0,0 +1,358 @@ +# audit_annotations.yml — curated judgment for the data-audit dashboard. +# +# The scanner (build_audit.py) derives every mechanical fact from the lecture +# repos themselves: who reads what, from where, by which URL form. What it +# cannot derive is judgment — what a file contains, how it came to exist, why +# a lecture calls a live API. That lives here, keyed by the scanner's output, +# and the scan warns (or fails with --strict) whenever a NEW reference appears +# with no entry, so this file cannot silently rot. +# +# Migrated datasets (lectures/*.yml manifests) do NOT get entries in +# `datasets:` — the manifest is their source of truth. +# +# provenance values (prior audit §7, feeding manual#108): +# verbatim third-party file as distributed; citation is the provenance +# constructed-committed built by a script/notebook that is committed somewhere +# constructed-lost construction documented but the pipeline never committed +# author-assembled hand-built by authors; provenance is lecture prose only +# toy invented in-lecture teaching table + +datasets: + SCF_plus_mini.csv: + description: SCF+ wealth/income survey extract + provenance: constructed-committed + note: built by generating_mini.md (executable MyST) in high_dim_data from SCF_plus.dta + SCF_plus_mini_no_weights.csv: + description: SCF+ extract, no survey weights + provenance: constructed-committed + note: > + variant of the SCF_plus_mini pipeline. Was pinned to feature branch + update_scf_noweights (critical flag in the 2026-07-15 audit); reads + high_dim_data main since lecture-python-intro#793 + acs_data_summary.csv: + description: ACS occupation summary + provenance: constructed-lost + note: construction (ACS filters, grouping, sorting) described in lecture prose only + assignat.xlsx: + description: Assignat issuance / price data, French Revolution + provenance: author-assembled + bbh_macro_quarterly.csv: + description: Macro quarterly series (Bhandari et al.) + provenance: constructed-lost + note: extracted from the paper's replication package (Zenodo DOI, CC BY 4.0); extraction not scripted + bbh_michigan_monthly.csv: + description: Michigan survey monthly series (Bhandari et al.) + provenance: constructed-lost + note: same Zenodo replication package; extraction not scripted + caron.npy: + description: French Revolution money balances (Caron) + provenance: author-assembled + note: hand-prepared NumPy array, no construction record + chapter_3.xlsx: + description: Hyperinflation tables, Sargent "Ends of Four Big Inflations" + provenance: author-assembled + note: hand-transcribed by authors + cities_brazil.csv: + description: Brazilian city populations + provenance: author-assembled + note: manual download from worldpopulationreview.com, documented in high_dim_data README + cities_us.csv: + description: US city populations + provenance: author-assembled + note: manual download from worldpopulationreview.com, documented in high_dim_data README + dataBHS.mat: + description: US consumption/income series, MATLAB replication bundle + provenance: verbatim + flags: [lectures-root] + dette.xlsx: + description: French government debt series + provenance: author-assembled + fig_3.xlsx: + description: French Revolution fiscal data + provenance: author-assembled + forbes-billionaires.csv: + description: Forbes billionaires list + provenance: constructed-committed + note: webscrape_forbes.ipynb in high_dim_data (Forbes API) + forbes-global2000.csv: + description: Forbes Global 2000 firm size + provenance: constructed-committed + note: webscrape_forbes.ipynb in high_dim_data (Forbes API) + fp.dta: + description: Treisman (2016) Russia billionaires replication data + provenance: verbatim + fred_data.csv: + description: FRED snapshot — GS1, GS5, GS10, DFII5, DFII10, USREC + provenance: constructed-lost + note: lecture names the FRED series; no build script anywhere + graph.txt: + description: 15-node weighted digraph for the shortest-path problem + provenance: toy + note: > + maintained as %%file blocks in intro, dp and jax; lecture-wasm instead + fetches intro's committed copy by URL (requests.get), making that copy + load-bearing — it was merely "shadowed" in the 2026-07-15 audit + hansen_jagannathan_1991_data.json: + description: Hansen–Jagannathan (1991) asset-returns bundle + provenance: constructed-lost + note: lecture documents 3 sources (FRED yields deflated by CPIAUCSL, …); no build script + hansen_singleton_1982_data.csv: + description: FRED-derived consumption/returns snapshot + provenance: constructed-committed + note: make_data.py + README beside the data (exact series and sample documented) + hansen_singleton_1983_data.csv: + description: FRED-derived snapshot + provenance: constructed-committed + note: make_data.py + README beside the data + japan_population_by_age.xlsx: + description: Japan population by age group (prob_dist bimodality example) + provenance: author-assembled + flags: [new-since-2026-07-15] + note: added by lecture-python-intro#790; upstream source not yet recorded + life-expectancy-vs-gdp-per-capita.csv: + description: Our World in Data, life expectancy vs GDP per capita + provenance: verbatim + note: OWID export, as downloaded + longprices.xls: + description: Long-run price levels (Sargent–Velde) + provenance: author-assembled + note: hand-transcribed by authors + maketable1.dta: + description: Acemoglu–Johnson–Robinson (2001) replication, table 1 + provenance: verbatim + maketable2.dta: + description: AJR (2001) replication, table 2 + provenance: verbatim + maketable4.dta: + description: AJR (2001) replication, table 4 + provenance: verbatim + mpd2020.xlsx: + description: Maddison Project Database 2020 (GDP per capita, long run) + provenance: verbatim + nom_balances.npy: + description: Nominal balances, French Revolution + provenance: author-assembled + note: hand-prepared NumPy array, no construction record + test_pwt.csv: + description: Penn World Table 7.0 extract + provenance: author-assembled + us_adult_heights.csv: + description: Heights of US adults, from NHANES (prob_dist motivating example) + provenance: constructed-lost + flags: [new-since-2026-07-15] + note: added by lecture-python-intro#790; NHANES-derived, extraction not scripted + usa-gini-nwealth-tincome-lincome.csv: + description: US Gini coefficients, wealth & income (long series) + provenance: constructed-committed + note: built by inequality/data.ipynb (committed beside it) from SCF_plus_mini + +# --------------------------------------------------------------------------- +# Live-API uses, keyed repo:lecture:access. `pedagogy` says WHY the call is +# live (prior audit §8): "lesson" = the fetch workflow is the teaching point; +# "mixed" = discovery workflow is taught but the plots could read a snapshot; +# "incidental" = the lecture just needs a series — snapshot-ready. +# --------------------------------------------------------------------------- +api: + lecture-python-intro:business_cycle:wbgapi: + series: NY.GDP.MKTP.KD.ZG, SL.UEM.TOTL.NE.ZS, FS.AST.PRVT.GD.ZS + pedagogy: mixed + note: teaches wb.series.info querying and metadata as part of the narrative + lecture-python-intro:business_cycle:pandas_datareader: + series: UNRATE, USREC, M0892AUSM156SNBR, UMCSENT, CPILFESL, INDPRO + pedagogy: mixed + note: snapshot pipeline already exists (data-lectures business_cycle_data.csv) but is unadopted + lecture-python-intro:commod_price:yfinance: + series: CT=F + pedagogy: incidental + lecture-python-intro:heavy_tails:wbgapi: + series: NY.GDP.MKTP.CD + pedagogy: incidental + lecture-python-intro:heavy_tails:yfinance: + series: AMZN, BTC-USD + pedagogy: incidental + lecture-python-intro:inequality:wbgapi: + series: SI.POV.GINI, NY.GDP.PCAP.KD + pedagogy: mixed + note: demonstrates searching for the Gini series ID with wbgapi + lecture-python-intro:prob_dist:yfinance: + series: AMZN, COST + pedagogy: incidental + lecture-python-programming:pandas:wbgapi: + series: GC.DOD.TOTL.GD.ZS + pedagogy: lesson + note: the section is literally "Using wbgapi and yfinance to Access Data" + lecture-python-programming:pandas:yfinance: + series: 11-ticker exercise list + pedagogy: lesson + lecture-python-programming:pandas:fredgraph.csv URL: + series: UNRATE + pedagogy: lesson + lecture-python.myst:kesten_processes:yfinance: + series: ^IXIC + pedagogy: incidental + lecture-python.myst:unemployment_linear:pandas_datareader: + series: UNRATE + pedagogy: incidental + lecture-python.myst:unemployment_shocks:pandas_datareader: + series: UNRATE + pedagogy: incidental + lecture-python-advanced.myst:doubts_or_variability:fredgraph.csv URL: + series: PCND, PCESV, DPCERD3Q086SBEA, CNP16OV + pedagogy: incidental + lecture-python-advanced.myst:subjective_beliefs_business_cycles:fredgraph.csv URL: + series: USREC + pedagogy: incidental + # lecture-wasm mirrors intro's sources; live APIs are the repo's core problem + # (pyodide cannot reach them — meta#143), so every entry here is a migration + # target, not an endorsement. + lecture-wasm:business_cycle:wbgapi: + series: NY.GDP.MKTP.KD.ZG, SL.UEM.TOTL.NE.ZS, FS.AST.PRVT.GD.ZS + pedagogy: mixed + note: mirror of intro's business_cycle; cannot run under pyodide + lecture-wasm:business_cycle:pandas_datareader: + series: UNRATE, USREC, M0892AUSM156SNBR, UMCSENT, CPILFESL, INDPRO + pedagogy: mixed + note: mirror of intro's business_cycle; cannot run under pyodide + lecture-wasm:commod_price:yfinance: + series: CT=F + pedagogy: incidental + note: mirror of intro's commod_price + lecture-wasm:heavy_tails:wbgapi: + series: NY.GDP.MKTP.CD + pedagogy: incidental + note: mirror of intro's heavy_tails + lecture-wasm:heavy_tails:pandas_datareader: + series: historical mirror — intro has since moved off pandas_datareader here + pedagogy: incidental + note: mirror of an older intro heavy_tails + lecture-wasm:heavy_tails:yfinance: + series: AMZN, BTC-USD + pedagogy: incidental + note: mirror of intro's heavy_tails + lecture-wasm:inequality:wbgapi: + series: SI.POV.GINI, NY.GDP.PCAP.KD + pedagogy: mixed + note: mirror of intro's inequality + lecture-wasm:prob_dist:yfinance: + series: AMZN, COST + pedagogy: incidental + note: mirror of intro's prob_dist + +# --------------------------------------------------------------------------- +# Committed data files no code-cell reads, keyed repo:path. +# kind: orphan (dead weight) | shadowed (a %%file cell regenerates it at +# build time) | superseded (lecture reads another copy) | mirror-orphan +# (lecture-wasm copy; the wasm lecture reads intro's URL instead) | +# exercise-download (prose link target — referenced, just not by a code cell) +# --------------------------------------------------------------------------- +committed_unreferenced: + lecture-python-intro:lectures/datasets/GDP_per_capita_world_bank.csv: + kind: orphan + note: no reference anywhere; the QuantEcon/data copy was dropped at Phase 2 + lecture-python-intro:lectures/datasets/Metadata_Country_API_NY.GDP.PCAP.CD_DS2_en_csv_v2_4770417.csv: + kind: orphan + note: metadata twin of the World Bank GDP-per-capita orphan + lecture-python-intro:lectures/datasets/fig_3.ods: + kind: orphan + note: source-format twin of fig_3.xlsx; referenced by nothing + lecture-python-programming:lectures/_static/lecture_specific/pandas/data/ticker_data.csv: + kind: orphan + note: no reference anywhere + lecture-python-programming:lectures/_static/lecture_specific/python_advanced_features/numbers.txt: + kind: shadowed + note: debugging lecture writes it with %%file + lecture-python-programming:lectures/_static/lecture_specific/python_advanced_features/test_table.csv: + kind: exercise-download + note: prose exercise link tells the reader to download this exact URL — keep + lecture-python-programming:lectures/_static/lecture_specific/python_foundations/test_table.csv: + kind: orphan + note: duplicate of the python_advanced_features copy + lecture-python-programming:lectures/_static/lecture_specific/python_foundations/us_cities.txt: + kind: shadowed + note: python_essentials writes it with %%writefile + lecture-python.myst:lectures/_static/lecture_specific/finite_markov/web_graph_data.txt: + kind: shadowed + note: finite_markov writes it with %%file + lecture-python.myst:lectures/web_graph_data.txt: + kind: shadowed + note: shadowed duplicate at lectures/ root + lecture-dp:lectures/graph.txt: + kind: shadowed + note: short_path regenerates it via %%file + lecture-dp:lectures/_static/lecture_specific/finite_markov/web_graph_data.txt: + kind: orphan + note: inherited from python.myst; no finite_markov data reference in this repo + lecture-dp:lectures/_static/lecture_specific/match_transport/acs_data_summary.csv: + kind: orphan + note: inherited copy, no consuming lecture in this repo + lecture-dp:lectures/_static/lecture_specific/mle/fp.dta: + kind: orphan + note: inherited from python.myst, no consuming lecture + lecture-dp:lectures/_static/lecture_specific/ols/maketable1.dta: + kind: orphan + note: inherited from python.myst, no consuming lecture + lecture-dp:lectures/_static/lecture_specific/ols/maketable2.dta: + kind: orphan + note: inherited from python.myst, no consuming lecture + lecture-dp:lectures/_static/lecture_specific/ols/maketable4.dta: + kind: orphan + note: inherited from python.myst, no consuming lecture + lecture-dp:lectures/_static/lecture_specific/pandas_panel/countries.csv: + kind: orphan + note: inherited copy; the consuming lectures live in other repos and now read data-lectures + lecture-dp:lectures/_static/lecture_specific/pandas_panel/employ.csv: + kind: orphan + note: inherited copy; the consuming lectures live in other repos and now read data-lectures + lecture-dp:lectures/_static/lecture_specific/pandas_panel/realwage.csv: + kind: orphan + note: inherited copy; the consuming lectures live in other repos and now read data-lectures + lecture-wasm:lectures/graph.txt: + kind: mirror-orphan + note: wasm's short_path fetches intro's committed graph.txt by URL, not this copy + lecture-wasm:lectures/_static/lecture_specific/inequality/usa-gini-nwealth-tincome-lincome.csv: + kind: mirror-orphan + note: wasm's inequality reads intro's copy by URL + lecture-wasm:lectures/_static/lecture_specific/simple_linear_regression/life-expectancy-vs-gdp-per-capita.csv: + kind: mirror-orphan + note: wasm's simple_linear_regression reads intro's copy by URL + lecture-wasm:lectures/datasets/GDP_per_capita_world_bank.csv: + kind: mirror-orphan + note: orphan in intro, mirrored into wasm + lecture-wasm:lectures/datasets/Metadata_Country_API_NY.GDP.PCAP.CD_DS2_en_csv_v2_4770417.csv: + kind: mirror-orphan + note: orphan in intro, mirrored into wasm + lecture-wasm:lectures/datasets/assignat.xlsx: + kind: mirror-orphan + note: wasm's french_rev reads intro's copy by URL + lecture-wasm:lectures/datasets/caron.npy: + kind: mirror-orphan + note: wasm's french_rev reads intro's copy by URL + lecture-wasm:lectures/datasets/chapter_3.xlsx: + kind: mirror-orphan + note: wasm's inflation_history reads intro's copy by URL + lecture-wasm:lectures/datasets/dette.xlsx: + kind: mirror-orphan + note: wasm's french_rev reads intro's copy by URL + lecture-wasm:lectures/datasets/fig_3.ods: + kind: mirror-orphan + note: orphan in intro, mirrored into wasm + lecture-wasm:lectures/datasets/fig_3.xlsx: + kind: mirror-orphan + note: wasm's french_rev reads intro's copy by URL + lecture-wasm:lectures/datasets/longprices.xls: + kind: mirror-orphan + note: wasm's inflation_history reads intro's copy by URL + lecture-wasm:lectures/datasets/mpd2020.xlsx: + kind: mirror-orphan + note: wasm's long_run_growth reads intro's copy by URL + lecture-wasm:lectures/datasets/nom_balances.npy: + kind: mirror-orphan + note: wasm's french_rev reads intro's copy by URL + continuous_time_mcs:lectures/old_stuff.txt: + kind: orphan + note: scratch file at lectures/ root; referenced by nothing + +# Scanner-visible targets that are not datasets (tutorial artifacts the +# lecture itself creates, code files, etc.) — suppressed from the audit. +ignore_targets: [] +ignore_committed: [] diff --git a/scripts/build_audit.py b/scripts/build_audit.py new file mode 100644 index 0000000..f91f585 --- /dev/null +++ b/scripts/build_audit.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Generate the data-audit dashboard — scan the Python-family lecture repos, +classify every data reference, and render the gh-pages site. + +Two stages, kept separate so the structured data is reusable (data-lectures#20): + + scan grep each lecture repo's origin/main for data references, classify + them, reconcile with the migrated-dataset manifests (lectures/*.yml) + and the migration tracker (migration.yml), and write audit.json + render read audit.json and write the static dashboard into site/ + + python scripts/build_audit.py scan --repos-dir ../ -o audit.json + python scripts/build_audit.py render --audit audit.json -o site/ + python scripts/build_audit.py all --repos-dir ../ -o site/ + +Sources of truth, in order: + - lectures/*.yml manifests — migrated datasets (class, license, consumers) + - migration.yml — migration lifecycle + PR provenance + - scripts/audit_annotations.yml — curated judgment for NOT-yet-migrated refs + (descriptions, provenance class, flags, API series). The scan FAILS when a + reference has neither a manifest nor an annotation, so new data reads in + the lecture repos surface loudly instead of rotting silently. + +Never use `gh search code` to find URLs — it tokenizes on `/` and `.` and +returns clean zeros. This script greps clones (workspace-lectures learned this +the hard way; see QuantEcon/data-lectures#20). +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections import defaultdict +from datetime import date +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parent.parent +LECTURES = REPO / "lectures" +ANNOTATIONS = REPO / "scripts" / "audit_annotations.yml" +MIGRATION = REPO / "migration.yml" + +# The 8 synced Python-family repos (workspace-lectures manifest.yml scope). +# Translations and the topic-based series are out of scope by decision. +SCAN_REPOS = [ + "lecture-python-intro", + "lecture-python-programming", + "lecture-python.myst", + "lecture-python-advanced.myst", + "lecture-jax", + "lecture-dp", + "lecture-wasm", + "continuous_time_mcs", +] + +ORG = "quantecon" +THIS_REPO_NAMES = {"data-lectures", "data"} # rename redirect +LEGACY_REPO_NAMES = {"lecture-python", "lecture-python.rst"} # retired pre-MyST repo + +DATA_EXT = {".csv", ".xlsx", ".xls", ".dta", ".npy", ".mat", ".json", ".txt", + ".parquet", ".pkl", ".zip", ".ods"} + +# Committed paths that carry a data extension but are not datasets. +COMMITTED_IGNORE = re.compile( + r"(^|/)(\.github|_notebook_repo|_build)/" + r"|(^|/)(requirements.*\.txt|robots\.txt|runtime\.txt|LICENSE.*|CNAME)$" + r"|(^|/)(_config\.yml|_toc\.yml)$" + r"|\.ipynb_checkpoints" +) + +CODE_CELL = re.compile(r"^```\{code-cell\}[^\n]*\n(.*?)^```", re.M | re.S) +SKIP_TAGS = re.compile(r"^:tags:.*(skip-execution|remove-cell)", re.M) +QUOTED_DATA = re.compile( + r"""['"]([^'"\s]+\.(?:csv|xlsx|xls|dta|npy|mat|json|txt|parquet|pkl|zip)) + (\?[^'"]*)?['"]""", + re.X, +) +URL_RE = re.compile(r"https?://[^\s'\")\]}>]+") +PCT_FILE = re.compile(r"^%%(?:write)?file\s+(?:-\w+\s+)?(\S+)", re.M) +# in-lecture writes: a later read of the same file is embedded, not a dataset +INLINE_WRITE = re.compile( + r"""open\(\s*['"]([^'"]+)['"]\s*,\s*(?:mode\s*=\s*)?['"][wa]b?['"] + |\.to_csv\(\s*['"]([^'"]+)['"] + |np\.save(?:txt)?\(\s*['"]([^'"]+)['"]""", + re.X, +) +# python string plumbing the resolver understands +BACKSLASH_JOIN = re.compile(r"\\\s*\n\s*") +ADJACENT_LITS = re.compile(r"""(['"])([^'"\n]*)\1\s*(?:\+\s*)?\n?\s*(['"])([^'"\n]*)\3""") +STR_ASSIGN = re.compile(r"^[ \t]*(\w+)\s*=\s*(f?)(['\"])([^'\"\n]*)\3\s*$", re.M) +VAR_PLUS_LIT = re.compile(r"(\w+)\s*\+\s*(['\"])([^'\"\n]*)\2") +FSTRING_LIT = re.compile(r"f(['\"])([^'\"\n]*\{\w+\}[^'\"\n]*)\1") +INTERP = re.compile(r"\{(\w+)\}") + + +def resolve_strings(text: str): + """Join continuations / adjacent literals, resolve simple string variables, + and return (normalized_text, fully-resolved string expressions).""" + text = BACKSLASH_JOIN.sub(" ", text) + prev = None + while prev != text: # merge chains of adjacent/`+`-joined literals + prev = text + text = ADJACENT_LITS.sub(lambda m: f"{m.group(1)}{m.group(2)}{m.group(4)}{m.group(1)}", text) + + values: dict[str, str] = {} + for _ in range(3): # a few passes resolve var → f-string → read chains + for m in STR_ASSIGN.finditer(text): + name, is_f, _, val = m.groups() + if is_f: + val = INTERP.sub(lambda i: values.get(i.group(1), i.group(0)), val) + if "{" not in val: + values[name] = val + + resolved = set(values.values()) + for m in FSTRING_LIT.finditer(text): + val = INTERP.sub(lambda i: values.get(i.group(1), i.group(0)), m.group(2)) + if "{" not in val: + resolved.add(val) + for m in VAR_PLUS_LIT.finditer(text): + var, _, lit = m.groups() + if var in values: + resolved.add(values[var] + lit) + return text, sorted(resolved) + +GH_URL_FORMS = [ + # (regex, canonical spelling label) + (re.compile(r"https?://github\.com/([^/]+)/([^/]+)/raw/refs/heads/([^/]+)/(.+)"), + "github.com/{org}/{repo}/raw/refs/heads/{ref}/…"), + (re.compile(r"https?://github\.com/([^/]+)/([^/]+)/raw/([^/]+)/(.+)"), + "github.com/{org}/{repo}/raw/{ref}/…"), + (re.compile(r"https?://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.+?)\?raw=true"), + "github.com/{org}/{repo}/blob/{ref}/…?raw=true"), + (re.compile(r"https?://raw\.githubusercontent\.com/([^/]+)/([^/]+)/refs/heads/([^/]+)/(.+)"), + "raw.githubusercontent.com/{org}/{repo}/refs/heads/{ref}/…"), + (re.compile(r"https?://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/(.+)"), + "raw.githubusercontent.com/{org}/{repo}/{ref}/…"), + (re.compile(r"https?://media\.githubusercontent\.com/media/([^/]+)/([^/]+)/([^/]+)/(.+)"), + "media.githubusercontent.com/media/{org}/{repo}/{ref}/…"), +] + +API_MARKERS = [ + # (regex on a code line, provider, access label) + (re.compile(r"\bwbgapi\b|\bwb\.(data|series|economy)\b"), "World Bank", "wbgapi"), + (re.compile(r"pandas_datareader|\bDataReader\b|web\.DataReader"), "FRED", "pandas_datareader"), + (re.compile(r"fredgraph\.csv"), "FRED", "fredgraph.csv URL"), + (re.compile(r"\byfinance\b|\byf\.(download|Ticker)\b"), "Yahoo Finance", "yfinance"), +] + + +def run(cmd: list[str], cwd: Path) -> str: + return subprocess.run(cmd, cwd=cwd, check=True, capture_output=True, + text=True).stdout + + +def git_show(repo_dir: Path, path: str) -> str: + return run(["git", "show", f"origin/main:{path}"], repo_dir) + + +# --------------------------------------------------------------------------- +# Scan +# --------------------------------------------------------------------------- + +def classify_url(url: str, consuming_repo: str): + """Classify a URL → (pattern, detail dict) or None if not a data URL.""" + scan_repos_l = {r.lower() for r in SCAN_REPOS} + for rx, form in GH_URL_FORMS: + m = rx.match(url) + if not m: + continue + org, repo, ref, path = m.groups() + if Path(path.split("?")[0]).suffix.lower() not in DATA_EXT: + return None # partial URL from an unresolved continuation, or non-data + repo_l = repo.lower().removesuffix(".git") + if org.lower() != ORG: + pattern = "external" + elif repo_l in THIS_REPO_NAMES: + pattern = "data-lectures" + elif repo_l == consuming_repo.lower(): + pattern = "own-repo" + elif repo_l in LEGACY_REPO_NAMES: + pattern = "legacy" + elif repo_l in scan_repos_l: + pattern = "sibling" + else: + pattern = "external" + return pattern, { + "org": org, "gh_repo": repo, "ref": ref, "path": path, + "url_form": form, + "branch_pinned": ref not in ("main", "master", "gh-pages"), + "lfs_media": "media.githubusercontent.com" in url, + } + if "fredgraph.csv" in url: + return "api", {"provider": "FRED", "access": "fredgraph.csv URL"} + ext = Path(url.split("?")[0]).suffix.lower() + if ext in DATA_EXT: + return "external-web", {"url_form": "non-GitHub host"} + return None + + +def scan_repo(name: str, repos_dir: Path): + repo_dir = repos_dir / name + if not (repo_dir / ".git").exists(): + sys.exit(f"error: {repo_dir} is not a git clone (run `make sync` " + f"in the workspace, or pass --repos-dir)") + sha = run(["git", "rev-parse", "--short", "origin/main"], repo_dir).strip() + tree = run(["git", "ls-tree", "-r", "--name-only", "origin/main"], + repo_dir).splitlines() + + lecture_files = [p for p in tree + if p.startswith("lectures/") and p.endswith(".md")] + committed = [p for p in tree + if Path(p).suffix.lower() in DATA_EXT + and not COMMITTED_IGNORE.search(p)] + + refs = [] # static/embedded reads + api_uses = [] # (lecture, provider, access) + for lf in lecture_files: + text = git_show(repo_dir, lf) + cells = "\n".join(c for c in CODE_CELL.findall(text) + if not SKIP_TAGS.search(c)) + if not cells: + continue + lecture = Path(lf).stem + cells, resolved = resolve_strings(cells) + + embedded_targets = {t for t in PCT_FILE.findall(cells) + if Path(t).suffix.lower() in DATA_EXT} + embedded_targets |= {t for g in INLINE_WRITE.findall(cells) + for t in g if t + and Path(t).suffix.lower() in DATA_EXT} + for tgt in sorted(embedded_targets): + refs.append({"repo": name, "lecture": lecture, "lecture_path": lf, + "file": Path(tgt).name, "target": tgt, + "pattern": "embedded"}) + + # candidate targets: raw URLs, resolved string expressions, quoted paths + candidates = [] + for url in URL_RE.findall(cells): + candidates.append(url.rstrip("'\".,;:")) + candidates += [r for r in resolved + if "://" in r or Path(r).suffix.lower() in DATA_EXT] + for m in QUOTED_DATA.finditer(cells): + if "{" in m.group(1): + continue # unresolved f-string piece + if re.search(r"[\w\)\]]\s*\+\s*['\"]$", cells[:m.start() + 1]): + continue # suffix of a var + 'literal' concat — resolved above + candidates.append(m.group(1)) + + seen = set() + for cand in candidates: + if cand in seen: + continue + seen.add(cand) + if cand.startswith(("http://", "https://")): + got = classify_url(cand, name) + if not got: + continue + pattern, detail = got + if pattern == "api": + api_uses.append((lecture, detail["provider"], detail["access"])) + continue + refs.append({"repo": name, "lecture": lecture, "lecture_path": lf, + "file": Path(detail.get("path", cand).split("?")[0]).name, + "target": cand, "pattern": pattern, **detail}) + else: + if Path(cand).suffix.lower() not in DATA_EXT: + continue + base = Path(cand).name + if base in {Path(t).name for t in embedded_targets}: + continue # read-back of an in-lecture write — Registry B + refs.append({"repo": name, "lecture": lecture, "lecture_path": lf, + "file": base, "target": cand, "pattern": "local-path"}) + + for rx, provider, access in API_MARKERS: + if rx.search(cells): + api_uses.append((lecture, provider, access)) + + # dedupe: one row per (lecture, pattern, target) + uniq, out = set(), [] + for r in refs: + key = (r["lecture"], r["pattern"], r["target"]) + if key not in uniq: + uniq.add(key) + out.append(r) + api = sorted({(l, p, a) for l, p, a in api_uses}) + return {"sha": sha, "refs": out, "committed": committed, + "api": [{"lecture": l, "provider": p, "access": a} for l, p, a in api]} + + +def committed_referenced(repo: str, committed: list[str], all_refs: list[dict]): + """Which committed data files does some lecture actually read?""" + referenced = set() + for path in committed: + base = Path(path).name + for r in all_refs: + if r["pattern"] == "embedded": + continue # a %%file cell shadows, it does not reference + if r["pattern"] == "local-path" and r["repo"] == repo: + tgt = r["target"].lstrip("./") + cands = {f"lectures/{tgt}", tgt, + str(Path(r["lecture_path"]).parent / tgt)} + if path in cands: + referenced.add(path) + elif r.get("gh_repo", "").lower() == repo.lower(): + if r.get("path", "").strip("/") == path: + referenced.add(path) + elif r["pattern"] == "local-path" and r["repo"] == repo and \ + Path(r["target"]).name == base: + referenced.add(path) + return referenced + + +def load_yaml(path: Path): + if not path.exists(): + return {} + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def load_manifests(): + manifests = {} + for p in sorted(LECTURES.glob("*.yml")): + m = load_yaml(p) + if isinstance(m, dict) and "filename" in m: + data_file = LECTURES / m["filename"] + m["_size"] = data_file.stat().st_size if data_file.exists() else None + manifests[m["filename"]] = m + return manifests + + +def scan(repos_dir: Path): + ann = load_yaml(ANNOTATIONS) + ignore = set(ann.get("ignore_targets", [])) + ignore_committed = set(ann.get("ignore_committed", [])) + datasets_ann = ann.get("datasets", {}) + api_ann = ann.get("api", {}) + orphan_ann = ann.get("committed_unreferenced", {}) + + manifests = load_manifests() + migration = load_yaml(MIGRATION) + + repos, all_refs, all_api = {}, [], [] + for name in SCAN_REPOS: + res = scan_repo(name, repos_dir) + res["refs"] = [r for r in res["refs"] if r["target"] not in ignore] + repos[name] = {"sha": res["sha"], "committed": res["committed"]} + all_refs += res["refs"] + for a in res["api"]: + all_api.append({"repo": name, **a}) + + # committed-but-unreferenced sweep + orphans = [] + n_committed = 0 + for name in SCAN_REPOS: + committed = [p for p in repos[name]["committed"] + if f"{name}:{p}" not in ignore_committed] + repos[name]["committed"] = committed + n_committed += len(committed) + used = committed_referenced(name, committed, all_refs) + for path in committed: + if path not in used: + note = orphan_ann.get(f"{name}:{path}", {}) + orphans.append({"repo": name, "path": path, + "kind": note.get("kind", "orphan"), + "note": note.get("note", "")}) + + # aggregate static refs into datasets keyed by basename + static = [r for r in all_refs if r["pattern"] not in ("embedded",)] + embedded = [r for r in all_refs if r["pattern"] == "embedded"] + by_file = defaultdict(list) + for r in static: + by_file[r["file"]].append(r) + + datasets, missing_ann = [], [] + for fname, rows in sorted(by_file.items()): + manifest = manifests.get(fname) + a = datasets_ann.get(fname, {}) + if not manifest and not a: + missing_ann.append(fname) + patterns = sorted({r["pattern"] for r in rows}) + migrated = "data-lectures" in patterns + fully_migrated = migrated and patterns == ["data-lectures"] + datasets.append({ + "file": fname, + "refs": rows, + "patterns": patterns, + "consumers": sorted({(r["repo"], r["lecture"]) for r in rows}), + "migrated": migrated, + "fully_migrated": fully_migrated, + "manifest": bool(manifest), + "description": (manifest or {}).get("title") or a.get("description", ""), + "provenance": ((manifest or {}).get("class") or a.get("provenance", "")), + "flags": a.get("flags", []), + "note": a.get("note", ""), + }) + + # API registry: mechanical detection enriched by annotations + api_rows, api_missing = [], [] + for u in all_api: + key = f"{u['repo']}:{u['lecture']}:{u['access']}" + a = api_ann.get(key) + if a is None: + api_missing.append(key) + a = {} + api_rows.append({**u, "series": a.get("series", ""), + "pedagogy": a.get("pedagogy", ""), + "note": a.get("note", "")}) + + # migration.yml vs reality: the check that keeps the tracker honest + mig_problems = [] + by_name = {d["file"]: d for d in datasets} + for fname, rec in (migration.get("datasets") or {}).items(): + d = by_name.get(fname) + status = rec.get("status") + if status in ("repointed", "final"): + if d is None: + mig_problems.append( + f"{fname}: marked {status} but no lecture reads it at all") + elif not d["fully_migrated"]: + stale = sorted({r["pattern"] for r in d["refs"]} - {"data-lectures"}) + mig_problems.append( + f"{fname}: marked {status} but consumers still read via {stale}") + elif status in ("pending", "landed") and d and d["migrated"]: + mig_problems.append( + f"{fname}: marked {status} but some consumer already reads data-lectures") + if status in ("landed", "repointed", "final") and fname not in manifests: + mig_problems.append(f"{fname}: marked {status} but has no manifest") + for fname, m in manifests.items(): + if fname not in (migration.get("datasets") or {}): + mig_problems.append(f"{fname}: has a manifest but no migration.yml record") + for wave in migration.get("pending") or []: + for fname in wave.get("datasets") or []: + d = by_name.get(fname) + if d and d["migrated"]: + mig_problems.append( + f"{fname}: in pending wave {wave.get('pilot')} but already " + f"read from data-lectures") + + audit = { + "generated": date.today().isoformat(), + "repos": {n: {"sha": repos[n]["sha"], + "n_committed": len(repos[n]["committed"])} + for n in SCAN_REPOS}, + "datasets": datasets, + "embedded": embedded, + "api": api_rows, + "orphans": orphans, + "manifests": {k: { + "title": m.get("title"), "class": m.get("class"), + "license": (m.get("license") or {}).get("name"), + "redistribution": (m.get("license") or {}).get("redistribution"), + "integrity": ((m.get("integrity") or {}).get("upstream") or {}).get("status"), + "builder_status": m.get("builder_status"), + "size": m.get("_size"), + "consumers": m.get("consumers") or [], + } for k, m in manifests.items()}, + "migration": migration, + "problems": { + "missing_annotations": sorted(missing_ann), + "missing_api_annotations": sorted(api_missing), + "migration_inconsistencies": mig_problems, + }, + "stats": { + "static_files": len(datasets), + "committed_files": n_committed, + "orphans": len(orphans), + "legacy_refs": sum(1 for r in static if r["pattern"] == "legacy"), + "migrated": sum(1 for d in datasets if d["fully_migrated"]), + "api_lectures": len({(u["repo"], u["lecture"]) for u in all_api}), + "url_forms": len({r.get("url_form") for r in static if r.get("url_form")}), + }, + } + return audit + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("stage", choices=["scan", "render", "all"]) + ap.add_argument("--repos-dir", type=Path, + default=REPO.parent, + help="directory containing the lecture repo clones " + "(default: this repo's parent, i.e. the workspace repos/)") + ap.add_argument("--audit", type=Path, default=REPO / "audit.json") + ap.add_argument("-o", "--out", type=Path, default=None) + ap.add_argument("--strict", action="store_true", + help="fail when a data reference has no annotation/manifest") + args = ap.parse_args() + + if args.stage in ("scan", "all"): + audit = scan(args.repos_dir) + out = args.out if (args.out and args.stage == "scan") else args.audit + out.write_text(json.dumps(audit, indent=1, default=str), encoding="utf-8") + probs = audit["problems"] + for key, items in probs.items(): + for it in items: + print(f"warning: {key}: {it}", file=sys.stderr) + print(f"wrote {out} — {audit['stats']['static_files']} static files, " + f"{audit['stats']['orphans']} orphans, " + f"{audit['stats']['api_lectures']} live-API lectures") + if args.strict and any(probs.values()): + return 1 + + if args.stage in ("render", "all"): + from render_audit import render # noqa: deferred import, same dir + audit = json.loads(args.audit.read_text(encoding="utf-8")) + site = args.out or (REPO / "site") + render(audit, site) + print(f"wrote {site}/") + + return 0 + + +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).parent)) + raise SystemExit(main()) diff --git a/scripts/render_audit.py b/scripts/render_audit.py new file mode 100644 index 0000000..3b60749 --- /dev/null +++ b/scripts/render_audit.py @@ -0,0 +1,919 @@ +#!/usr/bin/env python3 +"""Render the data-audit dashboard (site/) from audit.json. + +Invoked by build_audit.py's `render` stage; not usually run directly. The +design system evolves the 2026-07-15 audit artifact: same theme-aware palette, +pattern/provenance pills and table language, extended with a top nav, a landing +page with the migration meter, and the per-dataset lifecycle stepper. + +Chart colors are validated (dataviz six-checks) — meter series: +light #2c7a44/#1f6fb2, dark #42a065/#3f8ec7, neutral track for the remainder. +Pills are labeled (identity never rides on color alone). +""" +from __future__ import annotations + +import html +from pathlib import Path + +GH = "https://github.com" +DL = f"{GH}/QuantEcon/data-lectures" +RAW = f"{DL}/raw/main/lectures" +PREV_SNAPSHOT = "2026-07-15" + +PATTERN_META = { + # key: (pill css class, label, one-line description) + "data-lectures": ("p-dl", "data-lectures", "reads this repo's published tree — the target state"), + "own-repo": ("p-own", "own-repo URL", "fetches a file committed in its own repo via a GitHub raw URL"), + "local-path": ("p-local", "local path", "pd.read_csv('…') relative path — no URL; breaks in Colab/download"), + "sibling": ("p-sib", "sibling repo URL", "fetches another lecture repo's committed copy by URL"), + "external": ("p-ext", "external data repo", "QuantEcon/high_dim_data via raw and media (LFS) hosts"), + "legacy": ("p-legacy", "legacy-repo URL", "fetches from the retired pre-MyST lecture-python repo"), + "embedded": ("p-embed", "%%file embedded", "written by the lecture itself, then read back"), + "api": ("p-api", "live API", "fetched at build time from a third-party service"), +} + +PROVENANCE_META = { + "verbatim": ("p-own", "verbatim", "third-party file as distributed; the citation is the provenance"), + "constructed-committed": ("p-local", "constructed · builder committed", "built by a script/notebook that is committed"), + "constructed-lost": ("p-legacy", "constructed · pipeline lost", "construction documented but never committed"), + "author-assembled": ("p-ext", "author-assembled", "hand-built by authors; provenance is lecture prose only"), + "toy": ("p-embed", "toy", "invented in-lecture teaching data"), +} + +ORPHAN_KIND = { + "orphan": ("p-legacy", "orphan"), + "shadowed": ("p-embed", "shadowed"), + "superseded": ("p-legacy", "superseded"), + "mirror-orphan": ("p-embed", "mirror-orphan"), + "exercise-download": ("p-own", "exercise download"), +} + +PEDAGOGY_META = { + "lesson": ("p-local", "API is the lesson"), + "mixed": ("p-own", "mixed"), + "incidental": ("p-legacy", "incidental"), +} + +STATUS_STEPS = ["pending", "landed", "repointed", "final"] + +CSS = """ +:root { + --bg: #f6f8f9; --surface: #ffffff; --ink: #1f262c; --muted: #5a6673; + --line: #dde3e8; --accent: #1f6fb2; --accent-ink: #ffffff; + --warn: #9a6700; --warn-bg: #fdf3d7; --crit: #b3352b; --crit-bg: #fbe9e7; + --ok: #2c7a44; --ok-bg: #e2f2e6; + --own-bg: #e3eef8; --own-ink: #1a5c96; + --legacy-bg: #fdf3d7; --legacy-ink: #9a6700; + --ext-bg: #ede8f7; --ext-ink: #5b3f9e; + --local-bg: #e2f2e6; --local-ink: #23703a; + --embed-bg: #e9edf0; --embed-ink: #4d5a66; + --api-bg: #dff2f1; --api-ink: #0e6b66; + --dl-bg: #f7e6ef; --dl-ink: #a1336b; + --sib-bg: #f3ece1; --sib-ink: #7a5327; + --code-bg: #eef1f4; + --meter-a: #2c7a44; --meter-b: #1f6fb2; --meter-track: #e4e9ed; +} +:root[data-theme="dark"] { + --bg: #12171c; --surface: #1a2128; --ink: #dee5eb; --muted: #8a97a3; + --line: #2a333c; --accent: #5fa8dc; --accent-ink: #0e1418; + --warn: #e3b341; --warn-bg: #3a2f10; --crit: #f2857a; --crit-bg: #43201c; + --ok: #7fce97; --ok-bg: #16321e; + --own-bg: #17324a; --own-ink: #8ec4ea; + --legacy-bg: #3a2f10; --legacy-ink: #e3b341; + --ext-bg: #2c2344; --ext-ink: #b8a3e8; + --local-bg: #16321e; --local-ink: #7fce97; + --embed-bg: #242c33; --embed-ink: #9aa8b4; + --api-bg: #0f3230; --api-ink: #6fcdc6; + --dl-bg: #3c1e30; --dl-ink: #e08fc0; + --sib-bg: #342a1a; --sib-ink: #d3a95e; + --code-bg: #232b33; + --meter-a: #42a065; --meter-b: #3f8ec7; --meter-track: #2a333c; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #12171c; --surface: #1a2128; --ink: #dee5eb; --muted: #8a97a3; + --line: #2a333c; --accent: #5fa8dc; --accent-ink: #0e1418; + --warn: #e3b341; --warn-bg: #3a2f10; --crit: #f2857a; --crit-bg: #43201c; + --ok: #7fce97; --ok-bg: #16321e; + --own-bg: #17324a; --own-ink: #8ec4ea; + --legacy-bg: #3a2f10; --legacy-ink: #e3b341; + --ext-bg: #2c2344; --ext-ink: #b8a3e8; + --local-bg: #16321e; --local-ink: #7fce97; + --embed-bg: #242c33; --embed-ink: #9aa8b4; + --api-bg: #0f3230; --api-ink: #6fcdc6; + --dl-bg: #3c1e30; --dl-ink: #e08fc0; + --sib-bg: #342a1a; --sib-ink: #d3a95e; + --code-bg: #232b33; + --meter-a: #42a065; --meter-b: #3f8ec7; --meter-track: #2a333c; + } +} +* { box-sizing: border-box; } +body { + background: var(--bg); color: var(--ink); + font-family: "Avenir Next", "Segoe UI", system-ui, sans-serif; + font-size: 15px; line-height: 1.55; margin: 0; +} +main { max-width: 1100px; margin: 0 auto; padding: 28px 28px 80px; } +nav.top { + background: var(--surface); border-bottom: 1px solid var(--line); + position: sticky; top: 0; z-index: 10; +} +nav.top .inner { + max-width: 1100px; margin: 0 auto; padding: 10px 28px; + display: flex; align-items: center; gap: 22px; flex-wrap: wrap; +} +nav.top .brand { font-weight: 650; font-size: 15px; color: var(--ink); text-decoration: none; } +nav.top .brand span { color: var(--accent); } +nav.top a.item { + color: var(--muted); text-decoration: none; font-size: 14px; padding: 3px 2px; + border-bottom: 2px solid transparent; +} +nav.top a.item:hover { color: var(--ink); } +nav.top a.item.active { color: var(--accent); border-bottom-color: var(--accent); font-weight: 600; } +nav.top button.theme { + margin-left: auto; background: none; border: 1px solid var(--line); border-radius: 6px; + color: var(--muted); font-size: 13px; padding: 3px 10px; cursor: pointer; +} +header.doc { border-bottom: 2px solid var(--accent); padding-bottom: 20px; margin-bottom: 28px; } +.eyebrow { text-transform: uppercase; letter-spacing: 0.09em; font-size: 12px; color: var(--accent); font-weight: 600; margin: 18px 0 6px; } +h1 { font-size: 30px; line-height: 1.2; margin: 0 0 10px; font-weight: 650; text-wrap: balance; } +.meta { color: var(--muted); font-size: 13.5px; margin: 0; } +.meta code { font-size: 12px; } +h2 { font-size: 21px; margin: 44px 0 6px; font-weight: 650; text-wrap: balance; } +h2 .sec { color: var(--accent); font-variant-numeric: tabular-nums; margin-right: 8px; } +h3 { font-size: 16px; margin: 26px 0 4px; font-weight: 650; } +p { max-width: 76ch; } +p.lede { color: var(--muted); margin-top: 0; } +code, .mono { + font-family: ui-monospace, "SF Mono", "Cascadia Code", Menlo, Consolas, monospace; + font-size: 12.8px; +} +code { background: var(--code-bg); padding: 1px 5px; border-radius: 3px; } +a { color: var(--accent); } +.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 10px; margin: 22px 0 6px; } +.stat { background: var(--surface); border: 1px solid var(--line); border-radius: 6px; padding: 12px 14px; } +.stat b { display: block; font-size: 26px; font-weight: 650; font-variant-numeric: tabular-nums; line-height: 1.15; } +.stat span { font-size: 12.5px; color: var(--muted); line-height: 1.3; display: block; margin-top: 3px; } +.stat.warn b { color: var(--warn); } +.stat.crit b { color: var(--crit); } +.stat.ok b { color: var(--ok); } +.tablewrap { overflow-x: auto; background: var(--surface); border: 1px solid var(--line); border-radius: 6px; margin: 14px 0 8px; } +table { border-collapse: collapse; width: 100%; font-size: 13.5px; } +th { + text-align: left; font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.06em; + color: var(--muted); font-weight: 600; padding: 9px 12px; border-bottom: 1px solid var(--line); + white-space: nowrap; +} +td { padding: 8px 12px; border-bottom: 1px solid var(--line); vertical-align: top; } +tr:last-child td { border-bottom: none; } +td.num { font-variant-numeric: tabular-nums; text-align: right; } +tr.group td { + background: var(--code-bg); font-weight: 650; font-size: 12.5px; + text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted); padding: 6px 12px; +} +.pill { + display: inline-block; border-radius: 99px; padding: 1.5px 9px; font-size: 11.5px; + font-weight: 600; white-space: nowrap; letter-spacing: 0.01em; +} +.p-own { background: var(--own-bg); color: var(--own-ink); } +.p-legacy { background: var(--legacy-bg); color: var(--legacy-ink); } +.p-ext { background: var(--ext-bg); color: var(--ext-ink); } +.p-local { background: var(--local-bg); color: var(--local-ink); } +.p-embed { background: var(--embed-bg); color: var(--embed-ink); } +.p-api { background: var(--api-bg); color: var(--api-ink); } +.p-dl { background: var(--dl-bg); color: var(--dl-ink); } +.p-sib { background: var(--sib-bg); color: var(--sib-ink); } +.flag { color: var(--warn); font-weight: 600; font-size: 12.5px; } +.flag.crit { color: var(--crit); } +.flag.ok { color: var(--ok); } +.note { color: var(--muted); font-size: 13px; } +ul { padding-left: 22px; } +li { margin: 3px 0; } +.finding { background: var(--surface); border: 1px solid var(--line); border-left: 3px solid var(--accent); border-radius: 4px; padding: 12px 16px; margin: 10px 0; } +.finding.warn { border-left-color: var(--warn); } +.finding.crit { border-left-color: var(--crit); } +.finding.ok { border-left-color: var(--ok); } +.finding b { font-weight: 650; } +.finding p { margin: 4px 0 0; font-size: 13.5px; color: var(--muted); max-width: none; } +.legend { display: flex; flex-wrap: wrap; gap: 8px 14px; margin: 10px 0 4px; font-size: 12.5px; color: var(--muted); align-items: center; } +.legend .key { display: inline-flex; align-items: center; gap: 6px; } +.legend .swatch { width: 12px; height: 12px; border-radius: 3px; display: inline-block; } +/* migration meter — 2 series + neutral track, 2px gaps, direct labels */ +.meter { display: flex; height: 26px; border-radius: 6px; overflow: hidden; background: var(--meter-track); margin: 14px 0 6px; } +.meter .seg { height: 100%; } +.meter .seg + .seg { margin-left: 2px; } +.meter .seg.a { background: var(--meter-a); } +.meter .seg.b { background: var(--meter-b); } +/* bar list — one measure, single hue, labeled rows */ +.barlist { display: grid; grid-template-columns: max-content 1fr max-content; gap: 6px 12px; align-items: center; margin: 14px 0; } +.barlist .lbl { font-size: 13px; } +.barlist .bar { height: 14px; border-radius: 0 4px 4px 0; background: var(--accent); min-width: 2px; } +.barlist .val { font-variant-numeric: tabular-nums; font-size: 13px; color: var(--muted); } +/* landing nav cards */ +.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 14px; margin: 20px 0; } +.card { + display: block; background: var(--surface); border: 1px solid var(--line); border-radius: 8px; + padding: 16px 18px; text-decoration: none; color: var(--ink); +} +.card:hover { border-color: var(--accent); } +.card b { display: block; font-size: 16px; font-weight: 650; margin-bottom: 4px; color: var(--accent); } +.card span { font-size: 13px; color: var(--muted); } +/* lifecycle stepper */ +.pipeline { background: var(--surface); border: 1px solid var(--line); border-radius: 8px; padding: 16px 18px; margin: 12px 0; } +.pipeline .head { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; margin-bottom: 10px; } +.pipeline .head .name { font-weight: 650; font-size: 15px; } +.steps { display: flex; align-items: flex-start; } +.step { flex: 1; text-align: center; position: relative; min-width: 70px; } +.step .dot { + width: 14px; height: 14px; border-radius: 50%; margin: 0 auto 6px; + background: var(--surface); border: 2px solid var(--line); position: relative; z-index: 1; +} +.step.done .dot { background: var(--ok); border-color: var(--ok); } +.step.next .dot { border-color: var(--muted); } +.step::before { + content: ""; position: absolute; top: 6px; left: -50%; width: 100%; height: 2px; + background: var(--line); +} +.step:first-child::before { display: none; } +.step.done::before { background: var(--ok); } +.step .slbl { font-size: 11.5px; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted); font-weight: 600; } +.step.done .slbl { color: var(--ok); } +.step .sdetail { font-size: 12px; color: var(--muted); margin-top: 2px; line-height: 1.4; } +footer { + border-top: 1px solid var(--line); margin-top: 60px; padding-top: 16px; + color: var(--muted); font-size: 12.5px; +} +@media (max-width: 640px) { h1 { font-size: 24px; } main { padding: 20px 16px 60px; } } +""" + +THEME_JS = """ +(function () { + var saved = null; + try { saved = localStorage.getItem('audit-theme'); } catch (e) {} + if (saved === 'light' || saved === 'dark') document.documentElement.dataset.theme = saved; + window.toggleTheme = function () { + var cur = document.documentElement.dataset.theme; + if (!cur) cur = matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'; + var next = cur === 'dark' ? 'light' : 'dark'; + document.documentElement.dataset.theme = next; + try { localStorage.setItem('audit-theme', next); } catch (e) {} + }; +})(); +""" + + +def esc(s) -> str: + return html.escape(str(s if s is not None else "")) + + +def pill(kind_map: dict, key: str) -> str: + css, label = kind_map.get(key, ("p-embed", key))[:2] + return f'{esc(label)}' + + +def pattern_pill(p: str) -> str: + return pill(PATTERN_META, p) + + +def repo_link(repo: str) -> str: + return f'{esc(repo)}' + + +def lecture_link(repo: str, path: str, label: str) -> str: + return f'{esc(label)}' + + +def pr_link(ref: str) -> str: + """QuantEcon/repo#N → link.""" + if not ref or "#" not in str(ref): + return esc(ref) + repo, num = str(ref).rsplit("#", 1) + short = repo.split("/")[-1] + return f'{esc(short)}#{esc(num)}' + + +def issue_link(ref: str) -> str: + if not ref or "#" not in str(ref): + return esc(ref) + repo, num = str(ref).rsplit("#", 1) + short = repo.split("/")[-1] + return f'{esc(short)}#{esc(num)}' + + +def page(title: str, active: str, body: str, audit: dict) -> str: + nav_items = [("index.html", "overview", "Overview"), + ("migration.html", "migration", "Migration tracker"), + ("audit.html", "audit", "Full audit")] + nav = "".join( + f'{label}' + for href, key, label in nav_items) + shas = " · ".join(f"{esc(n)} {esc(v['sha'])}" + for n, v in audit["repos"].items()) + return f""" + + + + +{esc(title)} + + + + + +
+{body} +
+Generated {esc(audit["generated"])} by scripts/build_audit.py +from each repo's origin/main: {shas}. +Relates to {issue_link("QuantEcon/data-lectures#20")} · {issue_link("QuantEcon/meta#336")}. +
+
+ + +""" + + +# --------------------------------------------------------------------------- +# Page: overview (index.html) +# --------------------------------------------------------------------------- + +def migration_meter(audit: dict) -> str: + mig = audit["migration"] or {} + recs = mig.get("datasets") or {} + n_total = audit["stats"]["static_files"] + n_repointed = sum(1 for r in recs.values() if r.get("status") in ("repointed", "final")) + n_landed = sum(1 for r in recs.values() if r.get("status") == "landed") + queued = sorted({f for w in (mig.get("pending") or []) for f in w.get("datasets") or []}) + pct = lambda n: max(0.8, 100 * n / n_total) if n else 0 + segs = "" + if n_repointed: + segs += f'
' + if n_landed: + segs += f'
' + return f""" +

Migration progress

+ +
+repointed — consumers read data-lectures ({n_repointed}) +landed here, consumers not yet repointed ({n_landed}) +not migrated ({n_total - n_repointed - n_landed}, of which {len(queued)} queued in P3) +
+

Denominator: the {n_total} distinct static files lectures read today. The interim→final +URL cutover ({issue_link("QuantEcon/data-lectures#15")}) advances datasets from repointed to +final once data.quantecon.org is live; no dataset is final yet.

+""" + + +def pattern_barlist(audit: dict) -> str: + counts: dict[str, int] = {} + for d in audit["datasets"]: + for r in d["refs"]: + counts[r["pattern"]] = counts.get(r["pattern"], 0) + 1 + counts["embedded"] = len(audit["embedded"]) + counts["api"] = len({(u["repo"], u["lecture"]) for u in audit["api"]}) + order = [k for k in PATTERN_META if k in counts] + mx = max(counts.values()) + rows = "" + for k in order: + unit = "lectures" if k == "api" else "refs" + rows += (f'
{pattern_pill(k)}
' + f'
' + f'
{counts[k]} {unit}
') + return f""" +

Where lecture data comes from today

+
{rows}
+

Counts are lecture→file references (a file read by two lectures counts twice); +the live-API row counts lectures. Full taxonomy on the audit page.

+""" + + +def what_changed() -> str: + # The narrative diff vs the hand-built 2026-07-15 artifact. Static prose: + # it describes a fixed historical comparison, not live state. + return f""" +

What moved since the {PREV_SNAPSHOT} snapshot

+

The audit was first taken by hand on {PREV_SNAPSHOT}. Regenerating from live +repo state two days later, the picture has already changed — which is why this dashboard is generated.

+
4 datasets migrated and repointed. +

P1 moved lingcod_msy_recovery.csv (was a Colab-breaking local path); +P2 moved the pandas_panel trio realwage.csv / countries.csv / +employ.csv (5 of their 6 reads pointed at the retired legacy repo). All consumers +now read data-lectures. Details on the migration tracker.

+
Legacy-repo URLs: 8 → 0. +

The pre-MyST QuantEcon/lecture-python repo was renamed to +lecture-python.rst and archived; the P2 repoints plus the ols +maketable repoint (lecture-python.myst#966) and the retirement sweeps +(lecture-python-programming#576, lecture-python.myst#968) removed every live reference.

+
Branch-pinned refs: 1 → 0. +

The critical flag on SCF_plus_mini_no_weights.csv — pinned to a feature branch of +high_dim_data — was fixed by lecture-python-intro#793; it now reads main.

+
lecture-wasm joined the audit — and changed one fact. +

The WASM mirror carries its own copies of intro's data files (all unread — its lectures fetch +intro's copies by URL), and its short_path fetches intro's committed +graph.txt over the network. That file was "shadowed dead weight" in the prior audit; +it is now load-bearing.

+
2 new datasets appeared (lecture-python-intro#790). +

us_adult_heights.csv and japan_population_by_age.xlsx entered +prob_dist as local-path reads — the pattern P1 just migrated away from. New data +keeps arriving under the old conventions until the styleguide +({issue_link("QuantEcon/QuantEcon.manual#108")}) lands.

+""" + + +def render_index(audit: dict) -> str: + s = audit["stats"] + tiles = f""" +
+
{s["static_files"]}distinct static data files referenced by lectures
+
{s["migrated"]}migrated — every consumer reads data-lectures
+
{s["api_lectures"]}lectures fetching live third-party API data
+
{s["orphans"]}committed data files no lecture references
+
{s["legacy_refs"]}live URLs into the retired legacy repo (was 8)
+
{s["url_forms"]}GitHub URL spellings in use (was 6)
+
+""" + cards = f""" +
+Migration tracker → +Per-dataset lifecycle — what landed, what's repointed, which PRs did it, what's queued next. +Full audit → +Every data reference across the 8 repos: hosting patterns, registries, orphans, provenance, live-API analysis. +Dataset catalog ↗ +The migrated-only registry generated from the manifests — class, license, integrity, consumers. +
+""" + body = f""" +
+

QuantEcon · generated data audit · Python lecture family

+

Lecture data — audit & migration dashboard

+

Every dataset referenced by lecture source across the 8 synced Python-family repos, +regenerated from each repo's main by +a script — not maintained by hand. +Companion to the migration of lecture data into +QuantEcon/data-lectures.

+
+{tiles} +{migration_meter(audit)} +{cards} +{pattern_barlist(audit)} +{what_changed()} +""" + return page("QuantEcon lecture data — audit & migration dashboard", "overview", body, audit) + + +# --------------------------------------------------------------------------- +# Page: migration tracker +# --------------------------------------------------------------------------- + +def stepper(rec: dict, verified: bool) -> str: + status = rec.get("status", "pending") + reached = STATUS_STEPS.index(status) + detail = { + "pending": "", + "landed": pr_link(str((rec.get("landed") or {}).get("pr", ""))) + + f'
{esc((rec.get("landed") or {}).get("date", ""))}', + "repointed": "
".join(pr_link(str(r.get("pr", ""))) for r in rec.get("repoints") or []), + "final": ("awaits " + issue_link("QuantEcon/data-lectures#15")) if status != "final" + else esc((rec.get("cutover") or {}).get("date", "")), + } + steps = "" + for i, name in enumerate(STATUS_STEPS): + cls = "done" if i <= reached else ("next" if i == reached + 1 else "") + steps += (f'
' + f'
{name}
' + f'
{detail.get(name, "")}
') + check = ('✓ verified against today\'s scan' + if verified else + '✗ scan disagrees with this status') + return f'
{steps}

{check}

' + + +def render_migration(audit: dict) -> str: + mig = audit["migration"] or {} + recs = mig.get("datasets") or {} + manifests = audit["manifests"] + by_name = {d["file"]: d for d in audit["datasets"]} + + pipelines = "" + for fname, rec in recs.items(): + d = by_name.get(fname) + verified = bool(d and d["fully_migrated"]) if rec.get("status") in ("repointed", "final") else True + m = manifests.get(fname, {}) + consumers = ", ".join( + lecture_link(c["repo"].split("/")[-1], c["file"], f'{c["repo"].split("/")[-1]} · {Path(c["file"]).stem}') + for c in m.get("consumers", [])) + prior = rec.get("prior_pattern", "") + pipelines += f""" +
+
+{esc(fname)} +{esc(rec.get("pilot", ""))} +{pattern_pill(prior)} → {pattern_pill("data-lectures")} +file · +manifest +
+

{esc(m.get("title", ""))} — consumed by {consumers or "—"}

+{stepper(rec, verified)} +
+""" + + waves = "" + for w in mig.get("pending") or []: + files = "".join(f'
  • {esc(f)} — {esc((by_name.get(f) or {}).get("description", ""))}
    • ' + for f in w.get("datasets") or []) or \ + "
  • produces a new file here; no existing static file moves
    • " + tracking = " · ".join(issue_link(t) for t in w.get("tracking") or []) + waves += f""" +
    +{esc(w.get("pilot", ""))} — {esc(w.get("scope", ""))} +
      {files}
    +

    Tracking: {tracking}

    +
    +""" + + problems = audit["problems"]["migration_inconsistencies"] + consistency = ('
    Consistency check: clean.' + '

    migration.yml, the manifests and today\'s scan of every consumer repo agree.

    ' + if not problems else + '
    Consistency check: FAILING.
      ' + + "".join(f"
    • {esc(p)}
      • " for p in problems) + "
    ") + + n_re = sum(1 for r in recs.values() if r.get("status") in ("repointed", "final")) + queued = {f for w in (mig.get("pending") or []) for f in w.get("datasets") or []} + body = f""" +
    +

    Migration tracker · source of truth: migration.yml + lectures/*.yml

    +

    Datasets moving into data-lectures

    +

    Each dataset's position in the lifecycle +pending → landed → repointed → final, with the PRs that moved it. The build +verifies every status against a fresh scan of the consumer repos, so this page cannot +claim a migration that didn't happen.

    +
    +
    +
    {n_re}repointed — consumers read data-lectures today
    +
    {len(queued)}queued in pending waves (P3)
    +
    0final — on data.quantecon.org (gated on DNS, {issue_link("QuantEcon/data-lectures#15")})
    +
    +{consistency} +

    Migrated datasets

    +{pipelines} +

    Pending waves

    +

    Scoped but not landed. Files listed here still appear under their current +hosting pattern in the audit.

    +{waves} +

    How a dataset moves

    +

    pending — identified for migration; nothing in data-lectures yet. +landed — file + manifest merged here; consumers unchanged. +repointed — every consumer reads this repo's interim raw URL +(github.com/QuantEcon/data-lectures/raw/main/lectures/…). +final — every consumer reads https://data.quantecon.org/lectures/…; +the cutover is one sweep tracked in {issue_link("QuantEcon/data-lectures#15")}, gated on Phase 4 DNS.

    +""" + return page("Migration tracker — data-lectures", "migration", body, audit) + + +# --------------------------------------------------------------------------- +# Page: full audit +# --------------------------------------------------------------------------- + +def repo_of_record(d: dict) -> str: + """Group a dataset under the repo that owns the bytes being read.""" + for r in d["refs"]: + if r["pattern"] in ("own-repo", "local-path"): + return r["repo"] + if r["pattern"] == "sibling": + return r["gh_repo"] + if d["migrated"]: + return "data-lectures" + return d["refs"][0]["repo"] + + +def audit_registry_a(audit: dict) -> str: + groups: dict[str, list] = {} + for d in audit["datasets"]: + groups.setdefault(repo_of_record(d), []).append(d) + order = ["data-lectures", "lecture-python-intro", "lecture-python-programming", + "lecture-python.myst", "lecture-python-advanced.myst"] + rows = "" + for repo in [r for r in order if r in groups] + sorted(set(groups) - set(order)): + ds = groups[repo] + rows += f'{esc(repo)} — {len(ds)} files' + for d in sorted(ds, key=lambda x: x["file"]): + lecture_list = ", ".join(sorted({ + f'{r["lecture"]}' + (" (wasm)" if r["repo"] == "lecture-wasm" else "") + for r in d["refs"]})) + pills = " ".join(pattern_pill(p) for p in d["patterns"]) + flags = [] + if "local-path" in d["patterns"]: + flags.append('⚠ breaks in Colab/notebook download') + if any(r.get("lfs_media") for r in d["refs"]): + flags.append('via media.githubusercontent (LFS)') + if any(r.get("branch_pinned") for r in d["refs"]): + flags.append('✗ pinned to a non-default branch') + if "new-since-2026-07-15" in d.get("flags", []): + flags.append('new since the 2026-07-15 audit') + if "lectures-root" in d.get("flags", []): + flags.append('lives at lectures/ root') + if d["fully_migrated"]: + flags.append('✓ migrated') + note = d.get("note", "") + if note: + flags.append(f'{esc(note)}') + rows += (f'{esc(d["file"])}' + f'{esc(d["description"])}' + f'{esc(lecture_list)}' + f'{pills}' + f'{" ".join(flags)}') + return f""" +

    2Registry A — static dataset files ({len(audit["datasets"])})

    +

    Every distinct file a lecture reads, grouped by the repo that owns the bytes. +lecture-wasm mirrors intro's lectures and reads intro's copies by URL, so it appears as a +consumer, not an owner.

    +
    + +{rows} +
    FileContentsLecture(s)HostingFlags / notes
    +""" + + +def audit_patterns(audit: dict) -> str: + counts: dict[str, dict] = {} + for d in audit["datasets"]: + for r in d["refs"]: + c = counts.setdefault(r["pattern"], {"n": 0, "repos": set()}) + c["n"] += 1 + c["repos"].add(r["repo"].replace("lecture-python-", "").replace("lecture-", "")) + counts.setdefault("embedded", {"n": len(audit["embedded"]), "repos": { + e["repo"].replace("lecture-python-", "").replace("lecture-", "") for e in audit["embedded"]}}) + api_lects = {(u["repo"], u["lecture"]) for u in audit["api"]} + counts.setdefault("api", {"n": len(api_lects), "repos": { + r.replace("lecture-python-", "").replace("lecture-", "") for r, _ in api_lects}}) + rows = "" + for key, (css, label, desc) in PATTERN_META.items(): + if key not in counts: + continue + c = counts[key] + unit = " lectures" if key == "api" else "" + rows += (f'{pattern_pill(key)}{esc(desc)}' + f'{c["n"]}{unit}' + f'{esc(", ".join(sorted(c["repos"])))}') + + forms = sorted({r["url_form"] for d in audit["datasets"] for r in d["refs"] if r.get("url_form")}) + form_list = "".join(f'
  • {esc(f)}
    • ' for f in forms) + return f""" +

    1Hosting patterns in use

    +

    Eight distinct patterns. The 2026-07-15 audit found seven; adding lecture-wasm +surfaced the sibling repo URL pattern, and the migration added data-lectures +while retiring legacy-repo URL entirely.

    +
    + +{rows} +
    PatternWhat it looks likeRefsWhere
    +

    {len(forms)} URL spellings for "raw file on GitHub"

    + +

    Down from six — the blob/master…?raw=true spelling retired with the +legacy-repo refs. One canonical spelling is a styleguide question for +{issue_link("QuantEcon/QuantEcon.manual#108")}.

    +""" + + +def audit_registry_b(audit: dict) -> str: + by_file: dict[str, list] = {} + for e in audit["embedded"]: + by_file.setdefault(e["file"], []).append(e) + rows = "" + for fname, uses in sorted(by_file.items()): + series = " · ".join(sorted({u["repo"].replace("lecture-python-", "").replace("lecture-", "") + for u in uses})) + lects = ", ".join(sorted({u["lecture"] for u in uses})) + multi = ('— same data maintained in ' + f'{len({u["repo"] for u in uses})} repos' + if len({u["repo"] for u in uses}) > 1 else "") + rows += (f'{esc(fname)}{esc(lects)}' + f'{esc(series)} {multi}') + return f""" +

    3Registry B — data embedded in lecture source ({len(by_file)} files)

    +

    Written to the working directory by the lecture itself (%%file, +%%writefile, or an in-lecture open(…, 'w')), then read back. +Self-contained everywhere — including Colab — at the cost of data living inside narrative source.

    +
    + +{rows} +
    FileLecture(s)Series
    +""" + + +def audit_registry_c(audit: dict) -> str: + rows = "" + for u in sorted(audit["api"], key=lambda x: (x["provider"], x["repo"], x["lecture"])): + repo_short = u["repo"].replace("lecture-python-", "").replace("lecture-", "") + ped = pill(PEDAGOGY_META, u["pedagogy"]) if u.get("pedagogy") else "" + note = f'
    {esc(u["note"])}
    ' if u.get("note") else "" + rows += (f'{esc(u["provider"])} ' + f'{esc(u["access"])}' + f'{esc(u["series"])}' + f'{esc(u["lecture"])}{esc(repo_short)}' + f'{ped}{note}') + n_lects = len({(u["repo"], u["lecture"]) for u in audit["api"]}) + return f""" +

    4Registry C — live API data ({n_lects} lectures)

    +

    Fetched at build time from third-party services — the build-reproducibility +surface, and the blocker for the WASM/JupyterLite target (pyodide cannot reach most of these +APIs; that is lecture-wasm's core problem, {issue_link("QuantEcon/meta#143")}).

    +
    + +{rows} +
    Provider / accessSeries or tickersLectureSeriesWhy live
    +
    +{pill(PEDAGOGY_META, "lesson")} the fetch workflow is the teaching point — keep live +{pill(PEDAGOGY_META, "mixed")} discovery is taught, plots could read a snapshot +{pill(PEDAGOGY_META, "incidental")} just needs a series — snapshot-ready +
    +""" + + +def audit_reuse(audit: dict) -> str: + rows = "" + for d in audit["datasets"]: + repos = {c[0] for c in d["consumers"]} + if len(repos) > 1: + series = " + ".join(sorted(r.replace("lecture-python-", "").replace("lecture-", "") + for r in repos)) + how = ("all consumers read data-lectures — the target state" + if d["fully_migrated"] else + "wasm mirrors intro and reads intro's copy by URL" + if "lecture-wasm" in repos else "") + rows += (f'{esc(d["file"])}{esc(series)}' + f'{esc(how)}') + multi_embed = "" + by_file: dict[str, set] = {} + for e in audit["embedded"]: + by_file.setdefault(e["file"], set()).add(e["repo"]) + for fname, repos in sorted(by_file.items()): + if len(repos) > 1: + series = " + ".join(sorted(r.replace("lecture-python-", "").replace("lecture-", "") + for r in repos)) + multi_embed += (f'{esc(fname)}{esc(series)}' + f'duplicated as %%file blocks — {len(repos)} copies ' + f'of the same data to keep in sync') + return f""" +

    5Cross-series reuse

    +

    Files consumed by more than one repo. Before the migration the only true +cross-series files were the pandas_panel trio; the wasm mirror now multiplies every intro +dataset into a second consumer.

    +
    + +{rows}{multi_embed} +
    DatasetConsumersHow it's shared today
    +""" + + +def audit_orphans(audit: dict) -> str: + rows = "" + cur = None + for o in audit["orphans"]: + if o["repo"] != cur: + cur = o["repo"] + n = sum(1 for x in audit["orphans"] if x["repo"] == cur) + rows += f'{esc(cur)} — {n} files' + rows += (f'{esc(o["path"])}' + f'{pill(ORPHAN_KIND, o["kind"])}' + f'{esc(o["note"])}') + return f""" +

    6Committed but unreferenced data files ({len(audit["orphans"])})

    +

    Files in a repo's tree that no executed code cell reads. Orphan-sweep candidates +are tracked in {issue_link("QuantEcon/meta#337")}; shadowed and exercise-download +files are functional, just invisible to a URL-level audit.

    +
    +{pill(ORPHAN_KIND, "orphan")} dead weight — nothing reads it +{pill(ORPHAN_KIND, "shadowed")} a %%file cell regenerates it at build time +{pill(ORPHAN_KIND, "mirror-orphan")} wasm copy; the wasm lecture reads intro's URL +{pill(ORPHAN_KIND, "exercise-download")} prose link target — referenced, not by code +
    +
    + +{rows} +
    FileStateWhy
    +""" + + +def provenance_key(d: dict, manifests: dict) -> str: + """Canonical provenance class. Manifests use verbatim/constructed/dynamic + snapshot plus builder_status; a constructed dataset whose builder was never + recovered stays visible as pipeline-lost (AGENTS.md: that gap is a tracked + bug, not a resolved one).""" + p = d["provenance"] or "?" + if d["manifest"]: + m = manifests.get(d["file"]) or {} + if p in ("constructed", "dynamic snapshot"): + return ("constructed-lost" if m.get("builder_status") == "unrecovered" + else "constructed-committed") + return p + + +def audit_provenance(audit: dict) -> str: + counts: dict[str, int] = {} + for d in audit["datasets"]: + key = provenance_key(d, audit["manifests"]) + counts[key] = counts.get(key, 0) + 1 + tiles = "" + tone = {"constructed-lost": " warn", "author-assembled": " warn"} + for key, (css, label, desc) in PROVENANCE_META.items(): + if key in counts: + tiles += (f'
    {counts[key]}' + f'{esc(label)} — {esc(desc)}
    ') + rows = "" + for key, (css, label, desc) in PROVENANCE_META.items(): + ds = [d for d in audit["datasets"] + if provenance_key(d, audit["manifests"]) == key] + if not ds: + continue + rows += f'{esc(label)} — {len(ds)}' + for d in sorted(ds, key=lambda x: x["file"]): + src = ("manifest" if d["manifest"] else "audit annotation") + rows += (f'{esc(d["file"])}' + f'{esc(d.get("note") or d["description"])}' + f'{esc(src)}') + return f""" +

    7Provenance — how each file came to exist

    +

    The axis the manifest convention formalizes (verbatim / constructed / dynamic +snapshot, {issue_link("QuantEcon/meta#336")}). For migrated datasets this comes from their +manifests; for everything else, from the curated audit annotations.

    +
    {tiles}
    +
    + +{rows} +
    FileProvenanceRecorded in
    +""" + + +def audit_rules() -> str: + rules = [ + ("Default to snapshots.", + "A lecture reads data from a stable snapshot URL (data-lectures). Live API calls are " + "the exception and require a reason recorded in the lecture source."), + ("Live APIs are for teaching data access, not for getting data.", + "A live call is justified when the fetch workflow is itself the lesson (the pandas " + "lecture's wbgapi/yfinance sections) or when currency is the point. “The lecture " + "needs series X” is not a reason — that's what snapshots are for."), + ("Every live-API lecture gets a snapshot twin.", + "A refresh builder in data-lectures producing the snapshot (the business_cycle_data.csv " + "pattern, already prototyped). Breakage becomes a one-line URL switch, and the WASM " + "build always uses the twin — pyodide cannot reach the live APIs at all."), + ("Prefer direct CSV endpoints over wrapper libraries.", + "Where live access stays, fredgraph.csv-style URLs beat pandas_datareader-style " + "wrappers: one less dependency, and the git history shows the wrappers are what broke."), + ("Snapshots carry provenance metadata.", + "Source, series IDs, retrieval date, license and refresh cadence — the manifest schema " + "in this repo is the template; the provenance classes in §7 say which fields are " + "required."), + ] + findings = "".join(f'
    {i}. {esc(t)}

    {esc(b)}

    ' + for i, (t, b) in enumerate(rules, 1)) + return f""" +

    8Draft rules for styleguide/datasets.md

    +

    Distilled from the live-API pedagogy analysis (§4) and the migration pilots; +feeding {issue_link("QuantEcon/QuantEcon.manual#108")}.

    +{findings} +""" + + +def render_audit_page(audit: dict) -> str: + s = audit["stats"] + body = f""" +
    +

    Full audit · regenerated from each repo's main

    +

    Dataset registry — Python lecture family

    +

    Every dataset referenced by lecture source across the 8 synced repos. +The successor to the hand-built {PREV_SNAPSHOT} audit — same taxonomy, now generated.

    +
    +
    +
    {s["static_files"]}distinct static data files referenced
    +
    {s["committed_files"]}data files committed across the repos
    +
    {s["orphans"]}committed files no code cell references
    +
    {s["legacy_refs"]}legacy-repo URLs (8 on {PREV_SNAPSHOT})
    +
    {s["api_lectures"]}lectures fetching live API data
    +
    {s["migrated"]}datasets fully migrated to data-lectures
    +
    +{audit_patterns(audit)} +{audit_registry_a(audit)} +{audit_registry_b(audit)} +{audit_registry_c(audit)} +{audit_reuse(audit)} +{audit_orphans(audit)} +{audit_provenance(audit)} +{audit_rules()} +""" + return page("Full data audit — Python lecture family", "audit", body, audit) + + +def render(audit: dict, out_dir: Path) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "index.html").write_text(render_index(audit), encoding="utf-8") + (out_dir / "migration.html").write_text(render_migration(audit), encoding="utf-8") + (out_dir / "audit.html").write_text(render_audit_page(audit), encoding="utf-8") From db83b635955685415dc9ad7d323e25309e551dea Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 17 Jul 2026 13:00:43 +1000 Subject: [PATCH 2/4] Deploy the dashboard and the published tree to GitHub Pages .github/workflows/audit-dashboard.yml builds the dashboard from shallow clones of the 8 lecture repos and deploys the default Pages site: dashboard at /, the published lectures/ tree at /lectures/, audit.json alongside for reuse. Triggers: push to main, weekly cron, manual dispatch; on PRs touching the audit inputs it runs the strict build as a guardrail without deploying. Serving lectures/ on Pages pre-stages PLAN Phase 4: when the data.quantecon.org DNS question is resolved, only the custom domain changes. Checkout uses lfs: true so LFS-tracked files publish as bytes, never pointers. Part of #20. Co-Authored-By: Claude Fable 5 --- .github/workflows/audit-dashboard.yml | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/audit-dashboard.yml diff --git a/.github/workflows/audit-dashboard.yml b/.github/workflows/audit-dashboard.yml new file mode 100644 index 0000000..711dcc9 --- /dev/null +++ b/.github/workflows/audit-dashboard.yml @@ -0,0 +1,82 @@ +# The generated data-audit dashboard (data-lectures#20): scan the 8 Python- +# family lecture repos, verify migration.yml + the manifests against what the +# lectures actually read, render the dashboard, and deploy it to GitHub Pages +# together with the published lectures/ tree. +# +# The scan is strict: an unannotated data reference or a migration.yml status +# that disagrees with reality fails the build — the dashboard cannot rot into +# a hand-tended snapshot. On PRs the build runs as a guardrail; deploys happen +# only from main (push / weekly schedule / manual dispatch). + +name: audit-dashboard + +on: + push: + branches: [main] + schedule: + - cron: "17 5 * * 1" # weekly — lecture repos move under us + workflow_dispatch: + pull_request: + paths: + - migration.yml + - lectures/*.yml + - scripts/build_audit.py + - scripts/render_audit.py + - scripts/audit_annotations.yml + - .github/workflows/audit-dashboard.yml + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true # published files must be bytes, never LFS pointers + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - name: Clone the 8 lecture repos (shallow, main only) + run: | + mkdir -p repos + for repo in lecture-python-intro lecture-python-programming \ + lecture-python.myst lecture-python-advanced.myst \ + lecture-jax lecture-dp lecture-wasm continuous_time_mcs; do + git clone --quiet --depth 1 --branch main \ + "https://github.com/QuantEcon/$repo" "repos/$repo" + done + - name: Build the dashboard (strict — fails on unannotated refs or migration drift) + run: python scripts/build_audit.py all --strict --repos-dir repos -o site + - name: Assemble the Pages tree (dashboard at /, data at /lectures/) + run: | + mkdir -p _site + cp -r site/. _site/ + cp -r lectures _site/lectures + cp audit.json _site/audit.json + - uses: actions/upload-pages-artifact@v3 + with: + path: _site + + deploy: + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/configure-pages@v5 + with: + enablement: true + - id: deployment + uses: actions/deploy-pages@v4 From 52c621b3067d78ee4dc5b3bdc6f8b6591058e659 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 17 Jul 2026 13:00:43 +1000 Subject: [PATCH 3/4] Docs: record the dashboard, its update rules, and P2 completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README gains the audit-dashboard section; AGENTS.md gains the keep-it-truthful rules (migration.yml updated in the same PR as a landing/repoint; annotations for new references; never commit site/ or audit.json) and the expanded repo map; PLAN.md checks off the Pages deploy (default domain — custom domain stays open), adds the Phase 5 workflow entry, and marks pilot P2 complete (#17, QuantEcon/lecture-python-programming#578, QuantEcon/lecture-python.myst#973). Part of #20. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 17 ++++++++++++++++- PLAN.md | 5 +++-- README.md | 24 +++++++++++++++++++++++- scripts/README.md | 9 +++++++++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9a78ac4..aae1bdb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,16 @@ For the public data sources most snapshots come from (World Bank, FRED, Eurostat One exception: a `restricted` file the lectures have already served publicly may be cached here if it is marked `redistribution: restricted` with a `note` and logged for licence review in the migration-licensing tracker ([workspace-lectures#20](https://github.com/QuantEcon/workspace-lectures/issues/20)) — resolve it (permission, an open replacement, or removal) before `data.quantecon.org` is promoted as a public open-data host. +### The audit dashboard stays truthful + +The generated dashboard (`scripts/build_audit.py`, [#20](https://github.com/QuantEcon/data-lectures/issues/20)) verifies its three inputs against a fresh scan of the lecture repos, and the strict build **fails** when they drift. Keep them current in the same PR as the change that moves reality: + +- **Landing or repointing a dataset** → update its `migration.yml` record (status, PR refs, dates). A dataset marked `repointed` whose consumers still read an old URL — or the reverse — is a build failure, by design. +- **A new manifest** (`lectures/*.yml`) → add its `migration.yml` record; delete any stale entry for the same file in `scripts/audit_annotations.yml` (manifested datasets must not be annotated there). +- **The weekly scan flags an unannotated reference** (a lecture repo started reading a new file) → classify it and add an entry to `scripts/audit_annotations.yml`; that file holds judgment (description, provenance, why-live), never facts the scan can derive. + +`site/` and `audit.json` are generated — never commit them; CI rebuilds and deploys on every push to `main`. + ## Cross-repo hygiene - Changes here often pair with PRs in lecture repos and issues in `QuantEcon/meta`. In commit messages and PR bodies, **never place a GitHub closing keyword (`fixes`, `closes`, `resolves`, …) immediately before a cross-repo reference** like `QuantEcon/meta#336` — GitHub will auto-close the referenced issue when the commit lands on the default branch. Write "See QuantEcon/meta#336" or "Part of QuantEcon/meta#336". @@ -89,8 +99,13 @@ One exception: a `restricted` file the lectures have already served publicly may lectures/ # the published tree — flat, served at data.quantecon.org/lectures/ # 9 datasets + business_cycle's upstream metadata dumps # manifests live here as sidecars: .yml -scripts/ # builders — NOT published +scripts/ # builders + generators — NOT published business_cycle.py # writes business_cycle_data.csv into lectures/ + build_catalog.py # generates CATALOG.md from the manifests + build_audit.py # the audit dashboard: scan lecture repos → audit.json → site/ + render_audit.py # its render stage + audit_annotations.yml # curated judgment for not-yet-migrated data refs +migration.yml # migration lifecycle tracker (status + PR provenance per dataset) manifest-schema.yml # per-dataset manifest schema (strawman) requirements.txt PLAN.md # roadmap — start here diff --git a/PLAN.md b/PLAN.md index 5ed6f0b..1708f84 100644 --- a/PLAN.md +++ b/PLAN.md @@ -70,7 +70,7 @@ The sidecar naming uses the **full filename** (`mpd2020.xlsx.yml`, not `mpd2020. ### Phase 4 — Publishing -- [ ] GitHub Pages deploy of the published tree, **`lfs: true` at checkout** (else pointer files publish) +- [x] GitHub Pages deploy of the published tree, **`lfs: true` at checkout** (else pointer files publish) — landed 2026-07-17 with the audit dashboard (`.github/workflows/audit-dashboard.yml`, [#20](https://github.com/QuantEcon/data-lectures/issues/20)): the default `quantecon.github.io/data-lectures/` site serves the dashboard at `/` and the published tree at `/lectures/`. The custom domain below stays open - [ ] `data.quantecon.org` DNS + custom domain (an old NestJS box on AWS Sydney currently answers this name — investigate before repointing) - [ ] Verify `access-control-allow-origin: *` on served files (pyodide/JupyterLite, meta#143) - [ ] Monitor Pages soft limits (~1 GB site, 100 GB/month) @@ -84,6 +84,7 @@ The sidecar naming uses the **full filename** (`mpd2020.xlsx.yml`, not `mpd2020. Full automation: +- [x] Audit dashboard workflow ([#20](https://github.com/QuantEcon/data-lectures/issues/20), added 2026-07-17): `.github/workflows/audit-dashboard.yml` rebuilds the full-universe data audit + migration tracker from the 8 lecture repos' `main` (push to main / weekly / dispatch) and deploys it with the published tree to Pages. Strict mode fails the build on an unannotated data reference or a `migration.yml` status the scan contradicts - [ ] PR validation: manifest schema check + per-dataset invariant tests (expected columns/dtypes, row-count floor, date-range recency, no all-NaN columns, overlap-window agreement with the previous vintage) on every PR touching data. The schema decisions these tests force — column patterns for wide files, `known_nulls` exact-vs-ceiling, a canonical dtype vocabulary — are researched in [#14](https://github.com/QuantEcon/data-lectures/issues/14) - [ ] Retrofit `scripts/business_cycle.py` to the four-stage builder contract — it has fetch/transform/write today but **no validate stage**. Builder architecture and a copy-able template: [#14](https://github.com/QuantEcon/data-lectures/issues/14) - [ ] Scheduled refresh workflow for dynamic datasets — cron per cadence class, runs the builder (fetch → pre-process → validate → write), lands the result as a PR whose diff summary (rows added, date-range delta, overlap-window changes) is the review surface; low-risk series may auto-merge on green (first consumer: the UNRATE pilot, meta#338 P4) @@ -116,7 +117,7 @@ Verify that what this repo holds is actually the data it claims to be — agains The first end-to-end deployment: one dataset per hosting pattern, each the hardest representative of its class, carried through the full chain — layout, manifest, integrity check, publish, lecture repoint. Validates the convention empirically before anything is written into a standard. Sequence P1 → P2 → P3 → P4, each a small PR set (data repo + consuming lecture repos). - [x] **P1 — local-path static**: `lingcod_msy_recovery.csv` (`msy_fishery`, intro). Tests: single-PR green build under `-nW`, Colab-unchanged download, catalog metadata for an author-assembled file. **Complete 2026-07-17** — data half #12, repoint QuantEcon/lecture-python-intro#792 (lecture build green in the single repoint PR); served URL verified byte-identical to the manifest `sha256`; Colab holds by construction (the lecture now reads a public URL, where the old relative path was exactly what broke downloaded notebooks); metadata findings recorded in meta#338. **The repo is live from this merge** — the pyodide/CORS check below remains open -- [ ] **P2 — cross-series shared static**: the `pandas_panel` trio (`realwage.csv`, `countries.csv`, `employ.csv`), consumed by programming **and** python.myst. Tests: flat namespace with two consuming series, one data PR updating two lecture repos; retires 5 of the 8 legacy-repo references as a side effect +- [x] **P2 — cross-series shared static**: the `pandas_panel` trio (`realwage.csv`, `countries.csv`, `employ.csv`), consumed by programming **and** python.myst. Tests: flat namespace with two consuming series, one data PR updating two lecture repos; retires 5 of the 8 legacy-repo references as a side effect. **Complete 2026-07-17** — data half #17, repoints QuantEcon/lecture-python-programming#578 and QuantEcon/lecture-python.myst#973; both lecture repos' own stale copies deleted in the repoint PRs. Lifecycle recorded in `migration.yml` - [ ] **P3 — external-repo static with LFS**: the `heavy_tails` set (Forbes ×2, cities ×2) plus the SCF pair from `high_dim_data`. Tests: served URL makes the raw-vs-media LFS trap invisible, Pages handles LFS objects (`lfs: true`), builders (`webscrape_forbes.ipynb`, `generating_mini.md`) migrate alongside their data - [ ] **P4 — dynamic snapshot twin**: `UNRATE`, consumed today by 4 lectures across 3 repos via 2 access methods. Tests: the full dynamic template — manifest, four-stage builder, refresh-as-PR, canary catching an induced failure — plus the documented live-call ↔ snapshot switch mechanism - [ ] Verify each migrated URL with a pyodide/JupyterLite fetch (CORS, meta#143) diff --git a/README.md b/README.md index e7f291d..5b9fb53 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,31 @@ See the [draft convention](https://github.com/QuantEcon/QuantEcon.manual/pull/10 | Path | What | Published | | --- | --- | --- | | `lectures/` | the published tree — flat. Every dataset lives here, directly. No folder implies ownership by a lecture series: any lecture may consume any file | yes | -| `scripts/` | builders for constructed and dynamic datasets | no | +| `scripts/` | builders for constructed and dynamic datasets, plus the audit-dashboard generator | no | | `manifest-schema.yml` | the per-dataset manifest schema (strawman — see [`PLAN.md`](PLAN.md) Phase 2) | no | +| `migration.yml` | the migration lifecycle tracker — which PRs landed and repointed each dataset (transitional; archivable when the migration programme completes) | rendered | The tree is flat because the URL is the interface: `lectures/` maps to `data.quantecon.org/lectures/`, so a file can never be re-filed under a new owner and break its consumers. Anything outside `lectures/` is not served. + +## The audit dashboard + +A **generated** dashboard covering all data referenced by the 8 synced +Python-family lecture repos — the full-universe audit plus a per-dataset +migration tracker — deploys to this repo's GitHub Pages site alongside the +published `lectures/` tree ([data-lectures#20](https://github.com/QuantEcon/data-lectures/issues/20)). + +``` +python scripts/build_audit.py all --strict # scan + render into site/ +``` + +The scan greps each lecture repo's `main` (clones under `--repos-dir`; defaults +to this repo's parent, matching the workspace-lectures layout), classifies every +data reference, and reconciles three sources of truth: the manifests +(`lectures/*.yml`, migrated datasets), `migration.yml` (lifecycle + PR +provenance), and `scripts/audit_annotations.yml` (curated judgment for +not-yet-migrated references). A new data reference with no annotation, or a +migration status the scan contradicts, **fails the build** — the dashboard +cannot silently rot. CI rebuilds it on push to `main`, weekly, and on demand +(`.github/workflows/audit-dashboard.yml`). diff --git a/scripts/README.md b/scripts/README.md index bbe11e0..215296d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -14,3 +14,12 @@ bug. `business_cycle.py` is run by hand today and has no validate stage; retrofitting it to the four-stage contract (fetch → pre-process → validate → write) is PLAN Phase 5. + +## Not builders + +| Script | What | +| --- | --- | +| `build_catalog.py` | generates `CATALOG.md` from the manifests (`lectures/*.yml`) | +| `build_audit.py` | the audit dashboard: scans the 8 lecture repos, writes `audit.json`, renders `site/` | +| `render_audit.py` | the render stage of `build_audit.py` (HTML generation) | +| `audit_annotations.yml` | curated judgment for not-yet-migrated data references — the strict scan fails when a new reference has no entry here and no manifest | From fd75dfc0014f5f89376be61fc6ff86f1f75a4974 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Fri, 17 Jul 2026 13:17:44 +1000 Subject: [PATCH 4/4] Address Copilot review: taxonomy gap, status validation, notebook guard - render_audit.py: add the external-web pattern to PATTERN_META so a non-GitHub data URL would surface in the hosting-pattern table and distribution chart instead of being silently omitted (no current ref triggers it; the failure mode was the point) - build_audit.py: validate migration.yml status values in the scan-stage consistency check (an unknown status fails --strict with a clear message); render_audit.py's stepper raises a named error as backstop - build_audit.py: flag any .ipynb under lectures/ outside _static/ as an unscanned_notebooks problem. Every lecture source today is MyST .md and the only notebooks are builder/figure helpers (deliberately not consumers); if a notebook-format lecture ever appears, the build now refuses to miss it silently Co-Authored-By: Claude Fable 5 --- scripts/build_audit.py | 23 +++++++++++++++++++++-- scripts/render_audit.py | 10 ++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/build_audit.py b/scripts/build_audit.py index f91f585..aa8ff50 100644 --- a/scripts/build_audit.py +++ b/scripts/build_audit.py @@ -63,6 +63,9 @@ DATA_EXT = {".csv", ".xlsx", ".xls", ".dta", ".npy", ".mat", ".json", ".txt", ".parquet", ".pkl", ".zip", ".ods"} +# migration.yml lifecycle vocabulary (mirrored by the renderer's stepper) +MIGRATION_STATUSES = ("pending", "landed", "repointed", "final") + # Committed paths that carry a data extension but are not datasets. COMMITTED_IGNORE = re.compile( r"(^|/)(\.github|_notebook_repo|_build)/" @@ -211,6 +214,14 @@ def scan_repo(name: str, repos_dir: Path): lecture_files = [p for p in tree if p.startswith("lectures/") and p.endswith(".md")] + # Every lecture source today is MyST .md; the only notebooks under + # lectures/ are builder/figure helpers in _static/, which are deliberately + # NOT consumers (a builder reading raw data is provenance, not a data + # read). If a notebook-format lecture ever appears, refuse to miss it + # silently — surface it as a problem instead. + unexpected_notebooks = [p for p in tree + if p.startswith("lectures/") and p.endswith(".ipynb") + and "/_static/" not in p] committed = [p for p in tree if Path(p).suffix.lower() in DATA_EXT and not COMMITTED_IGNORE.search(p)] @@ -287,7 +298,8 @@ def scan_repo(name: str, repos_dir: Path): out.append(r) api = sorted({(l, p, a) for l, p, a in api_uses}) return {"sha": sha, "refs": out, "committed": committed, - "api": [{"lecture": l, "provider": p, "access": a} for l, p, a in api]} + "api": [{"lecture": l, "provider": p, "access": a} for l, p, a in api], + "unexpected_notebooks": unexpected_notebooks} def committed_referenced(repo: str, committed: list[str], all_refs: list[dict]): @@ -342,7 +354,7 @@ def scan(repos_dir: Path): manifests = load_manifests() migration = load_yaml(MIGRATION) - repos, all_refs, all_api = {}, [], [] + repos, all_refs, all_api, unscanned_nb = {}, [], [], [] for name in SCAN_REPOS: res = scan_repo(name, repos_dir) res["refs"] = [r for r in res["refs"] if r["target"] not in ignore] @@ -350,6 +362,7 @@ def scan(repos_dir: Path): all_refs += res["refs"] for a in res["api"]: all_api.append({"repo": name, **a}) + unscanned_nb += [f"{name}:{p}" for p in res["unexpected_notebooks"]] # committed-but-unreferenced sweep orphans = [] @@ -415,6 +428,11 @@ def scan(repos_dir: Path): for fname, rec in (migration.get("datasets") or {}).items(): d = by_name.get(fname) status = rec.get("status") + if status not in MIGRATION_STATUSES: + mig_problems.append( + f"{fname}: unknown status {status!r} — expected one of " + f"{', '.join(MIGRATION_STATUSES)}") + continue if status in ("repointed", "final"): if d is None: mig_problems.append( @@ -462,6 +480,7 @@ def scan(repos_dir: Path): "missing_annotations": sorted(missing_ann), "missing_api_annotations": sorted(api_missing), "migration_inconsistencies": mig_problems, + "unscanned_notebooks": sorted(unscanned_nb), }, "stats": { "static_files": len(datasets), diff --git a/scripts/render_audit.py b/scripts/render_audit.py index 3b60749..c84a251 100644 --- a/scripts/render_audit.py +++ b/scripts/render_audit.py @@ -27,6 +27,7 @@ "local-path": ("p-local", "local path", "pd.read_csv('…') relative path — no URL; breaks in Colab/download"), "sibling": ("p-sib", "sibling repo URL", "fetches another lecture repo's committed copy by URL"), "external": ("p-ext", "external data repo", "QuantEcon/high_dim_data via raw and media (LFS) hosts"), + "external-web": ("p-ext", "external web host", "fetches a data file from a non-GitHub host by URL"), "legacy": ("p-legacy", "legacy-repo URL", "fetches from the retired pre-MyST lecture-python repo"), "embedded": ("p-embed", "%%file embedded", "written by the lecture itself, then read back"), "api": ("p-api", "live API", "fetched at build time from a third-party service"), @@ -470,8 +471,13 @@ def render_index(audit: dict) -> str: # Page: migration tracker # --------------------------------------------------------------------------- -def stepper(rec: dict, verified: bool) -> str: +def stepper(fname: str, rec: dict, verified: bool) -> str: status = rec.get("status", "pending") + if status not in STATUS_STEPS: + raise SystemExit( + f"migration.yml: unknown status {status!r} for {fname} — expected one of " + f"{', '.join(STATUS_STEPS)} (the strict scan flags this too; run " + f"build_audit.py scan --strict)") reached = STATUS_STEPS.index(status) detail = { "pending": "", @@ -518,7 +524,7 @@ def render_migration(audit: dict) -> str: manifest

    {esc(m.get("title", ""))} — consumed by {consumers or "—"}

    -{stepper(rec, verified)} +{stepper(fname, rec, verified)} """