From 912ed33910e2928386a0aaa6c0de60c652fcf773 Mon Sep 17 00:00:00 2001 From: Ilia Gradina Date: Mon, 5 Oct 2026 21:38:12 +0300 Subject: [PATCH 1/5] fix(synthetic_control): put the treated unit in every in-space placebo's donor pool in_space_placebo() fit each placebo against the other J - 1 donors and never put the real treated unit in a placebo's pool, while the treated unit's own synthetic control uses all J donors. Unit j's RMSPE ratio then depended on which unit carried the treatment label, so the J + 1 ratios were not one set that random assignment permutes, and the permutation p-value was not exact. Abadie, Diamond & Hainmueller (2010, section 3.4) shift the treated unit into the donor pool. Each placebo now fits against every other unit, and test_sharp_null() / confidence_set() use the treated unit's null-imputed outcome Y_1 - f inside each placebo's synthetic control (Firpo & Possebom 2018, fn 10), weighted by the placebo's weight on it. Fixes #837 --- .../20261005-in-space-placebo-donor-pool.md | 23 +++ diff_diff/diagnostic_report.py | 4 +- diff_diff/guides/llms-full.txt | 2 +- diff_diff/synthetic_control.py | 53 +++-- diff_diff/synthetic_control_results.py | 84 +++++--- docs/methodology/REGISTRY.md | 13 +- .../papers/firpo-possebom-2018-review.md | 2 +- tests/test_methodology_synthetic_control.py | 184 +++++++++++++++++- 8 files changed, 304 insertions(+), 61 deletions(-) create mode 100644 changelog.d/20261005-in-space-placebo-donor-pool.md diff --git a/changelog.d/20261005-in-space-placebo-donor-pool.md b/changelog.d/20261005-in-space-placebo-donor-pool.md new file mode 100644 index 000000000..415c29488 --- /dev/null +++ b/changelog.d/20261005-in-space-placebo-donor-pool.md @@ -0,0 +1,23 @@ +### Fixed +- **`SyntheticControl` in-space placebo is now an exact permutation test**: + `in_space_placebo()` now fits each placebo against all the other units, the + real treated unit included, as Abadie, Diamond & Hainmueller (2010, §3.4, + p. 501) do ("shifting California to the donor pool"). It used to drop the + treated unit from every placebo's donor pool (the `SCtools::generate.placebos` + construction); that reference set changes with the treated unit, so the + permutation p-value was not exact. Every placebo that puts weight on the + treated unit now gets a different RMSPE ratio, so `placebo_p_value` and the + placebo rows of `get_placebo_df()` change on every such fit, by whole ranks; + the fit itself (`att`, weights, the treated unit's `rmspe_ratio`) does not + change. On the 10-unit no-effect panel of issue #837, 5 of the 10 + assignments move by one rank (3 up, 2 down) and the 10 p-values become + exactly 0.1, 0.2, ..., 1.0; before, 2 assignments gave `p = 0.1`. Over 300 + simulated 10-unit no-effect panels, with each unit treated in turn, the old + construction gave `p <= 0.1` for 11.7% of the assignments (Monte Carlo SE 0.25 + points); the new one gives exactly 10% on every panel. + `test_sharp_null()` and `confidence_set()` re-rank the new placebo gaps and + use the treated unit's null-imputed outcome `Y_1 - f` inside each placebo's + synthetic control (Firpo & Possebom 2018, fn 10), so their p-values and sets + change too. With exactly 2 donors each placebo is now fit against 2 units + (a real weight solve and `V` search instead of a forced `w = [1]`); fewer + than 2 donors still gives a NaN p-value. diff --git a/diff_diff/diagnostic_report.py b/diff_diff/diagnostic_report.py index 5ab3cacb6..b74d71aac 100644 --- a/diff_diff/diagnostic_report.py +++ b/diff_diff/diagnostic_report.py @@ -2966,8 +2966,8 @@ def _scm_native(self, r: Any) -> Dict[str, Any]: ), "too_few_donors": ( "in_space_placebo() was run but fewer than 2 donors are " - "available (each placebo is fit against the other donors); " - "placebo_p_value is NaN." + "available (with one, the treated unit and the donor are each " + "other's synthetic control); placebo_p_value is NaN." ), "all_placebos_failed": ( "in_space_placebo() was run but every donor refit failed to " diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 03ed4e2ab..af347c68e 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -675,7 +675,7 @@ scm.fit( ) -> SyntheticControlResults ``` -**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits against the other J-1 donors (the real treated unit is excluded from every placebo pool), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). +**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits it against all the other units (the real treated unit is in every placebo pool, ADH 2010 §3.4, so the permutation is exact), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). **Usage:** diff --git a/diff_diff/synthetic_control.py b/diff_diff/synthetic_control.py index bc61afbcc..642e64001 100644 --- a/diff_diff/synthetic_control.py +++ b/diff_diff/synthetic_control.py @@ -1861,6 +1861,8 @@ def _sharp_null_pvalue( f_post: np.ndarray, pre_denoms: Dict[Any, float], treated_id: Any, + *, + treated_weights: Optional[Dict[Any, float]] = None, ) -> Tuple[float, float, int]: """Permutation p-value for a common-effect sharp null (Firpo & Possebom Eq 13, φ=0, v=1). @@ -1870,13 +1872,20 @@ def _sharp_null_pvalue( to its gap path; ``pre_denoms`` maps each unit (treated + those donors) to its floored pre-denominator. Ties counted via ``>=`` so the p-value is conservative, matching ``in_space_placebo``. Returns ``(p, rmspe_f_treated, n_ref)``. + + ``treated_weights`` maps each placebo ``j`` to the weight ``w_j1`` its synthetic control + puts on the real treated unit, which is in every placebo's donor pool. Under the null the + treated unit's untreated outcome is ``Y_1 − f`` (Firpo & Possebom fn 10: the effect is known), + so placebo ``j``'s post gap moves by ``+w_j1·f`` and its residual is ``g_j − (1 − w_j1)·f``. + ``None`` means ``w_j1 = 0`` for every placebo. """ f_post = np.asarray(f_post, dtype=float) r1 = _rmspe_f_ratio(treated_gap, post_periods, f_post, pre_denoms[treated_id]) n_ref = 0 n_ge = 0 for j, g in placebo_gaps.items(): - rj = _rmspe_f_ratio(g, post_periods, f_post, pre_denoms[j]) + k = 1.0 if treated_weights is None else 1.0 - treated_weights[j] + rj = _rmspe_f_ratio(g, post_periods, k * f_post, pre_denoms[j]) n_ref += 1 if rj >= r1: n_ge += 1 @@ -1895,6 +1904,7 @@ def _invert_sharp_null( *, bounds: Optional[Tuple[float, float]] = None, n_grid: int = 200, + treated_weights: Optional[Dict[Any, float]] = None, ) -> Dict[str, Any]: """Invert the sharp-null RMSPE^f test over a one-parameter effect family. @@ -1910,6 +1920,8 @@ def _invert_sharp_null( ``p > gamma`` is the set -- correctly handling accepted tails, disjoint components, and the empty / unbounded cases. With explicit ``bounds`` a fixed ``linspace(*bounds, n_grid)`` grid is scanned instead (grid-limited membership). Each ``p`` is a pure O(J) re-ranking. + ``treated_weights`` is as in :func:`_sharp_null_pvalue`: placebo ``j``'s residual is + ``g_j − (1 − w_j1)·param·s_t``. Result dict: ``{lower, upper, contiguous, status, n_ref, point_estimate, grid}`` where ``status`` is ``"ran"`` / ``"empty"`` / ``"unbounded"``, ``point_estimate`` is the treated @@ -1920,8 +1932,9 @@ def _invert_sharp_null( if family not in ("constant", "linear"): raise ValueError(f"family must be 'constant' or 'linear', got {family!r}") n_post = len(post_periods) - # Family shape s_t and per-unit moments of A_u(param) = mean_t((g_ut - param*s_t)**2) - # = S2*param**2 - 2*P_u*param + Q_u (s_t = 1 constant / (t - T0) linear; S2 = mean(s**2)). + # Family shape s_t and per-unit moments of A_u(param) = mean_t((g_ut - k_u*param*s_t)**2) + # = k_u**2*S2*param**2 - 2*k_u*P_u*param + Q_u (s_t = 1 constant / (t - T0) linear; + # S2 = mean(s**2); k_u = 1 for the treated unit, 1 - w_j1 for placebo j). s = ( np.ones(n_post, dtype=float) if family == "constant" @@ -1929,12 +1942,18 @@ def _invert_sharp_null( ) S2 = float(np.mean(s**2)) if n_post else 0.0 - def moments(gap_path: Dict[Any, float], unit: Any) -> Tuple[float, float, float]: + def moments( + gap_path: Dict[Any, float], unit: Any, k: float = 1.0 + ) -> Tuple[float, float, float, float]: + # (k*P_u, Q_u, D_u, k**2*S2): the k-scaled moments of A_u. g = np.array([gap_path[p] for p in post_periods], dtype=float) - return float(np.mean(g * s)), float(np.mean(g**2)), pre_denoms[unit] + return k * float(np.mean(g * s)), float(np.mean(g**2)), pre_denoms[unit], k * k * S2 - P1, Q1, D1 = moments(treated_gap, treated_id) - placebo_mom = [moments(gp, j) for j, gp in placebo_gaps.items()] + P1, Q1, D1, _ = moments(treated_gap, treated_id) + placebo_mom = [ + moments(gp, j, 1.0 if treated_weights is None else 1.0 - treated_weights[j]) + for j, gp in placebo_gaps.items() + ] n_ref = len(placebo_mom) # The treated unit's own RMSPE-minimizing param: reported as the point estimate ONLY # (it is NOT assumed to maximize p -- see the exact-inversion comment below). @@ -1946,7 +1965,9 @@ def pval(param: float) -> float: # _sharp_null_pvalue / test_sharp_null (the squared form just avoids the sqrt). r1 = ((S2 * param - 2.0 * P1) * param + Q1) / D1 n_ge = sum( - 1 for (Pj, Qj, Dj) in placebo_mom if ((S2 * param - 2.0 * Pj) * param + Qj) / Dj >= r1 + 1 + for (Pj, Qj, Dj, S2j) in placebo_mom + if ((S2j * param - 2.0 * Pj) * param + Qj) / Dj >= r1 ) return (1 + n_ge) / (n_ref + 1) @@ -2023,8 +2044,8 @@ def quad_roots(a: float, b: float, c: float) -> List[float]: # 1[g_j(param) >= 0] == 1[RMSPE^param_j >= RMSPE^param_1], and breakpoints (where an # indicator flips) are the real roots of each g_j. coeffs = [ - (S2 * (D1 - Dj), -2.0 * (Pj * D1 - P1 * Dj), Qj * D1 - Q1 * Dj) - for (Pj, Qj, Dj) in placebo_mom + (S2j * D1 - S2 * Dj, -2.0 * (Pj * D1 - P1 * Dj), Qj * D1 - Q1 * Dj) + for (Pj, Qj, Dj, S2j) in placebo_mom ] raw_breaks: List[float] = [] for a, b, c in coeffs: @@ -2146,17 +2167,18 @@ def _placebo_fit_unit( unit: Any, donor_pool: List[Any], n_starts: int, -) -> Tuple[Optional[Tuple[Dict[Any, float], float]], str]: +) -> Tuple[Optional[Tuple[Dict[Any, float], float, Dict[Any, float]]], str]: """Refit a synthetic control for one (pseudo-)treated ``unit`` vs ``donor_pool``. Reuses the exact predictor build / standardization / weight solve / gap-path path as ``fit()`` — none of those helpers reads estimator (``self``) state, so - the refit is driven entirely by the snapshot. The real treated unit is never in - ``donor_pool`` (the caller passes the other ``J−1`` donors). + the refit is driven entirely by the snapshot. ``donor_pool`` may contain the real + treated unit: the in-space placebo passes every unit except ``unit`` (ADH 2010 §3.4). Returns ``(result, status)`` where ``status`` is one of: - - ``"ran"`` — ``result`` is ``(gap_path, rmspe_ratio)`` (a valid optimum). + - ``"ran"`` — ``result`` is ``(gap_path, rmspe_ratio, weights)`` (a valid optimum); + ``weights`` maps each ``donor_pool`` unit to its donor weight. - ``"infeasible"`` — ``result`` is ``None``; the refit is STRUCTURALLY unidentified (non-finite cells, or — under ``v_method="cv"`` — a re-aggregated window with no cross-donor variation for this pseudo-treated pool). Remedied by @@ -2266,7 +2288,8 @@ def _placebo_fit_unit( pre_gaps = np.array([gap_path[p] for p in snap.pre_periods], dtype=float) post_gaps = np.array([gap_path[p] for p in snap.post_periods], dtype=float) scale = float(np.max(np.abs(Z1))) if Z1.size else 0.0 - return (gap_path, _rmspe_ratio(pre_gaps, post_gaps, scale)), "ran" + weights = {d: float(w_d) for d, w_d in zip(donor_pool, w)} + return (gap_path, _rmspe_ratio(pre_gaps, post_gaps, scale), weights), "ran" # ============================================================================= diff --git a/diff_diff/synthetic_control_results.py b/diff_diff/synthetic_control_results.py index bca37931f..7748a74aa 100644 --- a/diff_diff/synthetic_control_results.py +++ b/diff_diff/synthetic_control_results.py @@ -316,6 +316,11 @@ def __post_init__(self) -> None: # anchor). Each value uses that unit's OWN pre-outcome scale; the pre window is # f-free so the denominator is grid-invariant. Small dict → survives pickling. self._placebo_pre_denoms: Optional[Dict[Any, float]] = None + # Each converged placebo's weight w_j1 on the real treated unit (which is in every + # placebo's donor pool), captured by in_space_placebo() so the sharp-null re-ranking + # can use the treated unit's untreated outcome Y_1 - f inside each placebo's + # synthetic control. Small dict → survives pickling. + self._placebo_treated_weights: Optional[Dict[Any, float]] = None # --- ADH 2015 §4 robustness diagnostics (opt-in, populated by --- # --- leave_one_out() / in_time_placebo()). Same panel-vs-scalar split as --- @@ -666,8 +671,8 @@ def summary(self, alpha: Optional[float] = None) -> str: ] elif status == "too_few_donors": reason = [ - "In-space placebo inference requires at least 2 donors (each " - "placebo is fit against the other donors); too few were", + "In-space placebo inference requires at least 2 donors (with one, the " + "treated unit and the donor are each other's synthetic control); too few were", "available. placebo_p_value is undefined. Inspect " "get_placebo_df().", ] elif status == "all_placebos_infeasible": @@ -863,18 +868,22 @@ def in_space_placebo( Section 2.4). Reassigns the treatment to each donor in turn, re-estimates a synthetic - control for that pseudo-treated donor against the OTHER donors, and ranks + control for that pseudo-treated donor against ALL the other units, and ranks the real treated unit's post/pre RMSPE ratio among all units. Populates ``placebo_p_value``, ``n_placebos``, ``n_failed`` and ``n_infeasible`` on this object (``rmspe_ratio`` — the treated unit's own ratio — is set at fit time) and returns the placebo distribution via :meth:`get_placebo_df`. - The real treated unit is **excluded from every placebo's donor pool**: its - post-period outcome is treatment-contaminated, so allowing a placebo to - load weight on it would bias the placebo gap. The ranking set is therefore - the ``J+1`` units ``{treated} ∪ {J placebos}``, with each placebo fit - against the other ``J-1`` donors (this matches the standard - ``SCtools::generate.placebos`` construction). The post/pre RMSPE ratio + The real treated unit is **in every placebo's donor pool**: each placebo is + fit against the other ``J`` units, as the treated unit is fit against its + ``J`` donors (ADH 2010 §3.4, journal p. 501: each placebo run reassigns the + intervention to a control state, "shifting California to the donor pool"). + All ``J+1`` statistics then come from one construction that does not depend + on which unit is treated, so under the null of no effect (and random + assignment) the permutation p-value is exact. (``SCtools::generate.placebos`` + instead drops the treated unit from every placebo's pool; that reference set + changes with the treated unit, so its p-value is not exact.) The ranking set + is the ``J+1`` units ``{treated} ∪ {J placebos}``. The post/pre RMSPE ratio normalizes by pre-treatment fit, which obviates the pre-fit-cutoff filtering of ADH Figures 5-7 (journal p. 502), so no pre-fit filter is offered — every converged placebo enters the rank. @@ -989,15 +998,16 @@ def in_space_placebo( self.n_infeasible = 0 self._placebo_gaps = {} self._placebo_pre_denoms = {} + self._placebo_treated_weights = {} self._placebo_status = "treated_fit_nonconverged" self._placebo_df = pd.DataFrame(rows, columns=self._PLACEBO_COLS) return self._placebo_df.copy() if n_donors < 2: warnings.warn( - "In-space placebo inference requires at least 2 donors (each " - f"placebo is fit against the other donors); only {n_donors} " - "available. placebo_p_value is NaN.", + "In-space placebo inference requires at least 2 donors (with one, the " + "treated unit and the donor are each other's synthetic control); only " + f"{n_donors} available. placebo_p_value is NaN.", UserWarning, stacklevel=2, ) @@ -1007,27 +1017,32 @@ def in_space_placebo( self.n_infeasible = 0 self._placebo_gaps = {} self._placebo_pre_denoms = {} + self._placebo_treated_weights = {} self._placebo_status = "too_few_donors" self._placebo_df = pd.DataFrame(rows, columns=self._PLACEBO_COLS) return self._placebo_df.copy() if n_donors == 2: warnings.warn( - "In-space placebo with 2 donors: each placebo is fit against a " - "single donor (degenerate weight w=[1]) with no V search, so the " - "permutation p-value is coarse (only 2 placebos enter the " - "reference set; the smallest attainable p-value is 1/3).", + "In-space placebo with 2 donors: only 2 placebos enter the " + "reference set, so the permutation p-value is coarse (the " + "smallest attainable p-value is 1/3).", UserWarning, stacklevel=2, ) placebo_gaps: Dict[Any, Dict[Any, float]] = {} + treated_weights: Dict[Any, float] = {} ranked_ratios: List[float] = [] n_failed = 0 n_infeasible = 0 + # Every unit but the placebo, the real treated unit included, in the pivot's sorted + # order: a unit's refit sees the same pool in the same order whichever unit is + # treated, as a fresh default-pool fit() treating it would. + units = list(snap.pivots[snap.outcome].columns) for j in donors: - pool = [d for d in donors if d != j] + pool = [u for u in units if u != j] fitted, fit_status = _placebo_fit_unit(snap, j, pool, n_starts_eff) if fitted is None: # Excluded from BOTH the numerator and the denominator (never rank a @@ -1051,8 +1066,9 @@ def in_space_placebo( } ) continue - gap_path_j, ratio_j = fitted + gap_path_j, ratio_j, weights_j = fitted placebo_gaps[j] = gap_path_j + treated_weights[j] = weights_j[snap.treated_id] pre_j = _mspe(gap_path_j, snap.pre_periods) post_j = _mspe(gap_path_j, snap.post_periods) ranked_ratios.append(ratio_j) @@ -1127,6 +1143,7 @@ def in_space_placebo( self.n_failed = int(n_failed) self.n_infeasible = int(n_infeasible) self._placebo_gaps = placebo_gaps + self._placebo_treated_weights = treated_weights # Classify a no-reference-set run by cause (mirrors in_time_placebo): a pure # solver failure ("all_placebos_failed", actionable via n_starts / tolerances) and # pure structural infeasibility ("all_placebos_infeasible", actionable via @@ -1306,7 +1323,7 @@ def leave_one_out(self, n_starts: Optional[int] = None) -> pd.DataFrame: } ) continue - gap_path_d, ratio_d = fitted + gap_path_d, ratio_d, _ = fitted loo_gaps[d] = gap_path_d att_d = float(np.mean([gap_path_d[p] for p in snap.post_periods])) atts.append(att_d) @@ -1647,7 +1664,7 @@ def in_time_placebo( } ) continue - gap_path, ratio = fitted + gap_path, ratio, _ = fitted in_time_gaps[t_f] = gap_path placebo_att = float(np.mean([gap_path[p] for p in snap_mod.post_periods])) rows.append( @@ -2249,8 +2266,8 @@ def _require_placebo_reference(self, n_starts: Optional[int]) -> None: "RMSPE ratio is not a valid optimum to rank against placebos" ), "too_few_donors": ( - "fewer than 2 donors are available (each placebo is fit against the " - "other donors)" + "fewer than 2 donors are available (with one, the treated unit and " + "the donor are each other's synthetic control)" ), "all_placebos_failed": ( "every donor refit failed to converge, so no placebo entered the " @@ -2312,9 +2329,12 @@ def test_sharp_null( """Test a sharp null hypothesis on the treatment-effect path (Firpo & Possebom 2018, §4.1). Tests ``H_0^f: α_{1,t} = f(t)`` for every post period (Eq 11) by subtracting the - hypothesized effect path ``f(t)`` from the post-period gaps of EVERY unit and - re-ranking the treated unit's modified RMSPE ratio against the placebo distribution - (Eqs 12–13 at ``φ = 0``, ``v = (1,…,1)`` — the equal-weights benchmark). The + hypothesized effect path ``f(t)`` from the treated unit's post-period gaps and + ``(1 − w_j1)·f(t)`` from each placebo ``j``'s, then re-ranking the treated unit's + modified RMSPE ratio against the placebo distribution (Eqs 12–13 at ``φ = 0``, + ``v = (1,…,1)`` — the equal-weights benchmark). ``w_j1`` is the weight placebo + ``j``'s synthetic control puts on the treated unit, whose untreated outcome under + the null is ``Y_1t − f(t)`` (Firpo & Possebom fn 10). The synthetic controls are NOT refit: this reuses the gap paths and per-unit denominators :meth:`in_space_placebo` already computed (run lazily here if needed). At ``effect = 0`` the p-value is identically the benchmark ``placebo_p_value`` @@ -2356,7 +2376,11 @@ def test_sharp_null( self._require_placebo_reference(n_starts) post_periods = list(self.post_periods) f_post = self._coerce_effect_path(effect, len(post_periods)) - assert self._placebo_gaps is not None and self._placebo_pre_denoms is not None + assert ( + self._placebo_gaps is not None + and self._placebo_pre_denoms is not None + and self._placebo_treated_weights is not None + ) p, r1, n_ref = _sharp_null_pvalue( self.gap_path, self._placebo_gaps, @@ -2364,6 +2388,7 @@ def test_sharp_null( f_post, self._placebo_pre_denoms, self.treated_unit, + treated_weights=self._placebo_treated_weights, ) return pd.Series( { @@ -2470,7 +2495,11 @@ def confidence_set( if float(bounds[1]) <= float(bounds[0]): raise ValueError(f"bounds must satisfy hi > lo, got {bounds!r}") self._require_placebo_reference(n_starts) - assert self._placebo_gaps is not None and self._placebo_pre_denoms is not None + assert ( + self._placebo_gaps is not None + and self._placebo_pre_denoms is not None + and self._placebo_treated_weights is not None + ) res = _invert_sharp_null( self.gap_path, self._placebo_gaps, @@ -2481,6 +2510,7 @@ def confidence_set( float(gamma), bounds=(None if bounds is None else (float(bounds[0]), float(bounds[1]))), n_grid=int(n_grid), + treated_weights=self._placebo_treated_weights, ) status = res["status"] if status == "unbounded": diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 9bbfb82cb..a5f61deba 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -3833,8 +3833,8 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Regression-weight extrapolation diagnostic** (`regression_weights()`, journal pp. 498-499): computes the implied donor weights `W^reg = X0a'(X0a X0a')^{-1} X1a` of the regression counterfactual `B̂'X_1` on the baseline fit's predictor matrices with an **intercept row prepended** (so `ι'W^reg = 1` under full row rank — matching ADH's "if a constant is included"). Regression is then also a weighting estimator summing to one, but with UNRESTRICTED weights: `w_reg < 0` or `> 1` flags donors it extrapolates on, outside the donors' convex hull the simplex-constrained SC never leaves (in ADH's application regression assigned negative weights to Greece/Italy/Portugal/Spain). Returns a per-donor `w_reg` / `w_sc` / `extrapolates` / `abs_extrapolation` table sorted by extrapolation magnitude; the reporting headline is `n_extrapolating`. Pure linear algebra (no re-fit), computed as the min-norm least-squares solution. **At full row rank** the system is exactly solvable, so `W^reg` is invariant to per-predictor row scaling — identical across the standardized (nested/custom) and raw (inverse_variance) predictor spaces (differing only for `cv`, which matches on validation-window predictors). Fails closed (for consistency with the other diagnostics) on a non-converged treated fit or `< 2` donors. **Note:** the sum-to-one property AND the row-scaling invariance both hold only at full ROW rank; when the intercept-augmented predictor matrix is rank-deficient (`k+1 > J` — e.g. the default per-period outcome lags when `T0 > J` — or collinear predictors) the reported `W^reg` is the MIN-NORM least-squares solution (a `UserWarning` fires; `_regw_rank_deficient` is set), `Σ W^reg` need not equal 1, and — because the inexact fit's residual metric is reweighted by row scaling — the min-norm `W^reg` is computed in the captured fit space and can itself differ across predictor spaces (still an informative extrapolation witness; within a fixed space, uniqueness of the least-squares solution depends on the predictor COLUMN rank — an overdetermined full-column-rank case has a unique solution that merely need not sum to 1). - **Sparse-SC subset search** (`sparse_synthetic_control(sizes=None, max_subsets=50000)`, journal pp. 506-507): for each target size `l < J`, EXHAUSTIVELY searches all `C(J, l)` donor subsets **holding V FIXED at the baseline fit's V** (ADH hold V fixed to make the combinatorial search tractable, footnote 20 — this reuses the captured fit-time predictor matrices + V, unlike `leave_one_out()`/`in_time_placebo()` which re-search V), refits the inner simplex solve per subset, and reports the best size-`l` synthetic (lowest pre-period outcome MSPE). Shows how the fit degrades and the ATT moves as the synthetic is forced sparse (`l = 4, 3, 2` degrade "moderately", `l = 1` much worse — a single-match design close to DiD, footnote 23). A `status="baseline"` row (the full fit) first, then one row per size with the winning subset's ids / weights / `pre_rmspe` / `att` / `delta_att`; `get_sparse_synthetic_control_gaps()` returns the per-size winner paths for the overlay. `sizes=None` sweeps `[1, 2, 3]` (clipped to `l < J`); a DEFAULTED size whose `C(J, l)` exceeds `max_subsets` is **skipped with a warning** (a defaulted call never raises), while an **explicitly** requested `l` with `C(J, l) > max_subsets` raises `ValueError` (the user-facing "exhaustive or raise" contract — no silent approximation). Non-converged subset solves are excluded and counted (`n_failed`); a size with no converged subset gets `status="all_subsets_failed"`. Fails closed (warn + baseline-only) on a non-converged treated fit or `< 2` donors. **Note:** pre-fit typically degrades as `l` shrinks but strict monotonicity is NOT guaranteed — subsets are ranked by the uniform-outcome pre-period MSPE while each subset's weights are V-optimal on the predictor objective. -**Confidence sets by test inversion (Firpo & Possebom 2018, §4, opt-in):** A confidence set for the treatment-effect path, built ON TOP of the in-space placebo and surfaced under `estimator_native_diagnostics` (the analytical inference contract is unchanged — see the non-analytical Note below). Under a common-effect sharp null `H_0^f: α_1t = f(t)` (Eq 11) the donor synthetic controls and the pre-period MSPE denominators do not change — only the post-period gaps shift by `f(t)` — so the test is a pure **re-ranking** of the gap paths `in_space_placebo()` already computed (no synthetic-control refits): -- **`test_sharp_null(effect, gamma=0.1)`** forms the modified RMSPE ratio `RMSPE^f_j = sqrt(mean_post((α̂_jt − f(t))²) / pre_denom_j)` for every unit (Eq 12) and the permutation p-value `p^f = (1 + #{converged placebos j : RMSPE^f_j ≥ RMSPE^f_1}) / (n_ref + 1)` (Eq 13 at `φ=0`, `v=(1,…,1)` — the equal-weights benchmark). `effect` is a scalar (a constant-in-time effect) or a length-`n_post` array (an arbitrary post-period path `f(t)` — e.g. an intervention cost path or a theory-predicted shape). At `f≡0` this is identically the in-space `placebo_p_value` (Eq 5 = Eq 13), held bit-for-bit by reusing each unit's floored pre-period denominator persisted from the placebo run (the pre window is `f`-free, so the denominator is grid-invariant; the floor scale is **per-unit** `max|Z1_j|`). +**Confidence sets by test inversion (Firpo & Possebom 2018, §4, opt-in):** A confidence set for the treatment-effect path, built ON TOP of the in-space placebo and surfaced under `estimator_native_diagnostics` (the analytical inference contract is unchanged — see the non-analytical Note below). Under a common-effect sharp null `H_0^f: α_1t = f(t)` (Eq 11) the donor weights and the pre-period MSPE denominators do not change (the weights use pre-period data only) — only the post-period gaps shift: the treated unit's untreated outcome is `Y_1t − f(t)`, and because the treated unit is in every placebo's donor pool, placebo `j`'s synthetic control moves by `−w_j1·f(t)` (`w_j1` = its weight on the treated unit; Firpo & Possebom fn 10: "under sharp null hypotheses, the treatment effect is known") — so the test is a pure **re-ranking** of the gap paths `in_space_placebo()` already computed (no synthetic-control refits): +- **`test_sharp_null(effect, gamma=0.1)`** forms the modified RMSPE ratio `RMSPE^f_j = sqrt(mean_post((α̂_jt − k_j·f(t))²) / pre_denom_j)` for every unit (Eq 12), with `k_1 = 1` for the treated unit and `k_j = 1 − w_j1` for placebo `j` (its gap under the imputed `Y_1t − f(t)` is `α̂_jt + w_j1·f(t)`, from which Eq 12 subtracts `f(t)`) and the permutation p-value `p^f = (1 + #{converged placebos j : RMSPE^f_j ≥ RMSPE^f_1}) / (n_ref + 1)` (Eq 13 at `φ=0`, `v=(1,…,1)` — the equal-weights benchmark). `effect` is a scalar (a constant-in-time effect) or a length-`n_post` array (an arbitrary post-period path `f(t)` — e.g. an intervention cost path or a theory-predicted shape). At `f≡0` this is identically the in-space `placebo_p_value` (Eq 5 = Eq 13), held bit-for-bit by reusing each unit's floored pre-period denominator persisted from the placebo run (the pre window is `f`-free, so the denominator is grid-invariant; the floor scale is **per-unit** `max|Z1_j|`). - **`confidence_set(family="constant"|"linear", gamma=0.1, bounds=None, n_grid=200)`** inverts that test over a one-parameter family: `"constant"` → `f(t)=c` (Eq 15), a confidence **interval** for a constant effect (Eq 16); `"linear"` → `f(t)=c̃·(t−T0)` with the 1-based post-period index (Eq 17), a confidence **set** over the slope `c̃` (Eq 18). Membership is the paper's **strict** `p^c > γ` (Eq 14 — see the boundary Note). With `bounds=None` the set is recovered **EXACTLY**: `p^c` is a piecewise-constant step function (each placebo's indicator flips only at the real roots of `A_j(c)·D_1 = A_1(c)·D_j`, a quadratic in `c`), so the placebo breakpoints partition the line and `p` is evaluated once per induced open interval AND at each breakpoint — where a tie under `≥` can lift `p` above γ, so an isolated accepted singleton (a tangent / co-located root) is captured. The accepted set is the union of accepted intervals/points, with **no centering or monotonicity assumption** (a poor-pre-fit treated unit can have its accepted region in the tails, not around the point estimate). `bounds=(lo,hi)` instead scans a fixed grid (grid-limited membership). The summary is on `effect_confidence_set` (`status ∈ {"ran","empty","unbounded"}`) and the full grid on `get_confidence_set_df()`. **Fail-closed:** `γ < 1/(J+1)` ⇒ `p^c > γ` for every `c` ⇒ `"unbounded"` (`±inf` endpoints + warning — the discrete-granularity point, fn 8); a treated unit lacking the best pre-fit can give a one-sided unbounded edge; if no interval or breakpoint is accepted the set is `"empty"` (NaN endpoints); a non-contiguous accepted region (disjoint components / an isolated singleton) reports the `[lower, upper]` hull with `contiguous=False` + a warning; and `< 2` donors / a non-converged treated fit / an unpickled result (no placebo reference set) raise `ValueError`. **Scope:** sensitivity weights (`φ≠0`, Eqs 7–9), the general test-statistic menu `θ¹–θ⁵` (Eq 19), one-sided (§7's signed-`t` statistic), and the multiple-outcome / multiple-treated extensions (§6) are **deferred** (flagged in the paper review checklist). **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in):** Valid p-values for a hypothesized post-period effect trajectory and pointwise confidence intervals — what the placebo / test-inversion paths cannot give. Surfaced under `estimator_native_diagnostics`; the analytical `se`/`t_stat`/`p_value`/`conf_int` stay NaN (separate permutation object — see the non-analytical Note). Under a sharp null `H0: θ = θ0` the counterfactual treated outcomes are imputed (`Y^N_{1t} = Y_{1t} − θ0_t`, `t>T0`; pre unchanged), a **time-permutation-invariant proxy** is fit UNDER THE NULL on **all** periods, and the statistic `S_q(û) = ((1/√T*)·Σ_{t>T0}|û_t|^q)^{1/q}` (CWZ §2.2) is referred to its permutation distribution (eq 2) by reshuffling residuals over time: @@ -3858,7 +3858,8 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Deviation from R:** `standardize="none"` disables predictor standardization entirely; R `Synth` always scales by the predictor SD. Provided for diagnostics; changes the geometry of the `V` objective. - **Note:** predictor rows support only **equal-weight** linear combinations of pre-period values — `mean` (`k_s = 1/T0`), `sum` (`k_s = 1`), and per-period outcome lags (identity, a single `k_s = 1`). ADH (2010) §2.3 defines the general form `Ȳ_i^{K_m} = Σ_s k_s Y_is` with *arbitrary* weights `k_s`; this release does NOT accept user-supplied non-uniform `K_m` weight vectors (and `median` and other non-linear aggregations are intentionally excluded). The supported set still spans the standard `Synth::dataprep` `predictors.op` + `special.predictors` usage; arbitrary-weight `K_m` is a deferred extension. - **Deviation from R:** predictor/outcome **aggregation fails closed on any non-finite (NaN/inf) cell**, whereas R `Synth::dataprep` hardwires `na.rm=TRUE` (aggregating over the observed cells of a partially-missing window). The fail-closed contract is deliberate: na-dropping silently aggregates different period subsets across units, yielding incomparable predictors with no warning. The analyst must restrict `predictor_window` / `special_predictors` / `pre_period_outcomes` periods (and the outcome panel) to where each variable is observed; both partially- and fully-missing windows raise `ValueError`. Only the row *ordering* matches `dataprep`, not the missing-data handling. -- **Note (in-space placebo donor pool):** the real treated unit is **excluded from every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J−1` donors, never the real treated unit (whose post-period is treatment-contaminated). The ranking set is still the `J+1` units {treated} ∪ {J placebos}. ADH 2010 §2.4 does not spell out placebo donor-pool composition; this matches the standard `SCtools::generate.placebos` construction (rotate the pseudo-treated identity through the donor pool; the original treated unit is never re-added as a donor). +- **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015). The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, so with the default donor pool it reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. +- **Deviation from R (`SCtools::generate.placebos`):** SCtools drops the treated unit from every placebo's pool, fitting each placebo against the other `J−1` donors. That reference set changes with the treated unit, so its p-value is not exact: on the 10-unit no-effect panel of `test_in_space_placebo_is_exact_permutation`, leaving the treated unit out of the placebo pools gives `p ≤ 0.2` for 3 of the 10 assignments. Every placebo that puts weight on the treated unit has a different RMSPE ratio here than in SCtools, so `placebo_p_value` can differ by whole ranks. - **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — raise `n_starts` on `in_space_placebo()` or re-fit with a larger `optimizer_options['maxiter']`). - **Note (in-space / leave-one-out infeasible vs failed split):** an excluded `in_space_placebo()` / `leave_one_out()` refit is attributed to one of two causes, mirroring the split `in_time_placebo` already reports. A **solver non-convergence** (a truncated inner `W` or outer `V` search) is tallied in `n_failed` (`_loo_n_failed`) with a `status="failed"` row; a **structural cv infeasibility** — under `v_method="cv"`, the pseudo-treated donor pool (in-space) or reduced pool (leave-one-out) is indistinguishable in a re-aggregated CV window, so the weights are unidentified (the window-information gate above; `_outer_solve_V_cv` returns a structural sentinel that `_placebo_fit_unit` threads out) — is tallied in `n_infeasible` (`_loo_n_infeasible`) with a `status="infeasible"` row. Both are excluded from the rank / ATT range identically, so `placebo_p_value` / `n_placebos` are **unchanged** by the attribution — only the diagnostic accounting is refined. `_placebo_status` / `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. `all_refits_infeasible` / `all_refits_unusable`) codes for a no-usable-refit run that is purely structural or a mix, and `DiagnosticReport` surfaces the machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. `n_infeasible` is 0 for the non-`cv` `v_method`s (no structural-identification gate). The remedy differs by cause: a structural infeasibility needs different predictors / `v_cv_t0` / donor pool, NOT a larger `n_starts` / `inner_max_iter`. - **Note (RMSPE-ratio floor):** the reported `rmspe_ratio = sqrt(MSPE_post / MSPE_pre)` floors the pre-period MSPE denominator at a scale-aware `1e-8 · max(|pre-outcomes|, 1)²` (before the square root) so a (near-)perfect pre-fit (`pre-MSPE → 0`) yields a large-but-FINITE ratio rather than `inf`/`nan` (which would corrupt the rank). Ties (`ratio_j ≥ treated_ratio`) are counted, making the p-value conservative. Mirrors the `_fit_tol` poor-fit guard. @@ -3866,7 +3867,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Note (confidence set is non-analytical — `conf_int` stays NaN):** the Firpo & Possebom test-inversion `effect_confidence_set` is a permutation set at level `1−γ`, kept DELIBERATELY SEPARATE from the analytical `conf_int` (which stays `(NaN, NaN)` — classic SCM has no Wald interval). It is parametrized by `γ` (not the estimator's `alpha`, and granular in `1/(J+1)`), can be a *set* rather than an interval (the linear family) and can be unbounded or non-contiguous, so it cannot be coerced into the `(lo, hi)` `conf_int` tuple without breaking the `safe_inference` NaN-consistency contract. `se`/`t_stat`/`p_value`/`conf_int`/`is_significant` all stay at their NaN state; the set is surfaced only via `confidence_set()` / `get_confidence_set_df()` / `effect_confidence_set` (and `estimator_native_diagnostics`) — mirroring how `placebo_p_value` is kept off `p_value`. - **Note (test-inversion boundary convention — strict `p^f > γ`):** Firpo & Possebom's inequalities are non-uniform at `p = γ` — the RMSPE-based tests reject at `p < γ` (Eqs 5/9/13), the general-statistic test rejects at `p ≤ γ` (Eq 19), and the confidence set is the **strict** `p^f > γ` (Eq 14), so Eq 14's set is NOT the exact complement of the Eq 13 rejection region (they differ at `p^f = γ`). Because the permutation p-value is discrete (a multiple of `1/(J+1)`), `p = γ` is reachable, so this implementation pins Eq 14's strict `p^f > γ` for set membership (a `p = γ` value is **excluded**) and documents it (matches the `firpo-possebom-2018-review.md` §4.2 boundary note). - **Note (test-inversion set construction is an implementation choice):** Firpo & Possebom §4.2 gives the set definitions (Eqs 14/16/18) but does NOT prescribe how to enumerate the set. The default is **exact piecewise-constant breakpoint inversion** — `p^c` is constant between the real roots of the placebo-vs-treated comparison quadratics, so evaluating `p` per induced interval and at each breakpoint recovers the set exactly (no grid resolution / shape assumption); the optional fixed `bounds=` grid is the grid-limited alternative. Either is OUR choice (the paper leaves it unspecified) — a documented deviation. (`n_grid` controls only the returned inspection table, not membership, when `bounds=None`.) -- **Note (test-inversion validation — no R anchor):** R `Synth` has no test-inversion confidence-set function, and the authors' Code Ocean replication capsule was not consulted (paper-sourced only). The confidence sets are validated by (a) **self-consistency** — `test_sharp_null(0)` equals the in-space `placebo_p_value` exactly (transitively R-anchored via the Basque placebo parity), (b) a **numpy oracle** re-implementing Eqs 12–14 on hand-built gap paths (incl. the strict `p = γ` boundary and the per-unit floor), and (c) a **coverage simulation** (a constant-effect DGP; the `1−γ` set covers the truth at ≈ `1−γ`). +- **Note (test-inversion validation — no R anchor):** R `Synth` has no test-inversion confidence-set function, and the authors' Code Ocean replication capsule was not consulted (paper-sourced only). The confidence sets are validated by (a) **self-consistency** — `test_sharp_null(0)` equals the in-space `placebo_p_value` exactly, (b) a **numpy oracle** re-implementing Eqs 12–14 on hand-built gap paths (incl. the strict `p = γ` boundary, the per-unit floor and each placebo's weight on the treated unit), (c) a **brute-force refit** of every placebo on the null-imputed panel (`Y_1t − f(t)`), and (d) a **coverage simulation** (a constant-effect DGP; the `1−γ` set covers the truth at ≈ `1−γ`). - **Note (in-time placebo windowing — TRUNCATE):** ADH 2015 §4 says to re-estimate the in-time placebo "with the same predictors lagged accordingly." Because `diff_diff`'s predictor specs reference **absolute** periods, the in-time placebo re-cuts them by TRUNCATION: pre-period-outcome predictors become the pre-`t_f` outcomes, and covariate / special-predictor windows are intersected with the pre-`t_f` window; a window lying ENTIRELY in the held-out region `[t_f, T0)` is **dropped** (surfaced in the `n_dropped_specs` column + an aggregated warning), and `custom_v` is subset in lockstep with the surviving specs. For an outcome-predictor fit (the R-anchorable case) TRUNCATE is identical to ADH's "lag" — both equal a manual `Synth::synth` re-run with `time.optimize.ssr` cut at `t_f`. The held-out window never enters the fit (the placebo's `all_periods` is the pre-fake + post-fake span; the true post-treatment periods are excluded entirely), so there is no "peeking." This concrete convention is NOT spelled out in ADH 2015 (which gives only the qualitative "lag accordingly"). - **Note (in-time placebo requires ≥2 pre-fake periods):** the in-time placebo treats a date with fewer than 2 pre-fake periods as `status="infeasible"` (the default sweep starts at the 3rd pre-period). This is DELIBERATELY stricter than the base estimator's `T0 ≥ 1` allowance (which permits a single-pre-period fit but warns that nested-`V` selection is unreliable): an auto-swept placebo date with a single pre-fake period is a trivially-matchable, non-credible pre-fit, so it is dropped rather than surfaced as a `ran` placebo (mirrors `SyntheticDiD.in_time_placebo`'s `i ≥ 2` rule). A date whose surviving `custom_v` has zero mass after truncation is likewise infeasible (not a convergence failure). - **Note (leave-one-out weight floor):** ADH 2015 §4 leave-one-out omits "each donor that received positive weight." This implementation drops each donor with **reportable** weight — above the `1e-6` interpretability floor (`synthetic_control._MIN_REPORT_WEIGHT`), i.e. exactly the donors in `donor_weights` — rather than every strictly-positive weight. A donor with `0 < w ≤ 1e-6` is numerical dust whose removal moves the ATT by ~its weight (its `delta_att` would be ~0, an uninformative row), and the floor keeps the LOO table aligned with the reported donor support. The drop-set is **frozen at fit time** on the fit snapshot (`weighted_donor_ids`), so `leave_one_out()` is immune to post-fit mutation of the presentation-level `donor_weights` dict. @@ -3878,7 +3879,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Note (conformal validity caveats — Remark 2; documented, not auto-corrected):** CWZ requires a **large `T0`** (drives exactness; `T*` may be short) and that `{u_t}` be stationary and **conditionally** heteroscedastic at most — **unconditional** heteroscedasticity / non-stationarity of the shocks invalidates the basic procedure (Remark 2), as does a change in the shock distribution under the intervention (Assumption 1 invariance). No standardizing filter is applied by default; the analyst must pre-filter if these are suspected. A `T*≥T0` fit emits a validity-caveat `UserWarning`. - **Note (conformal validation — no R anchor):** the authors state "all computations in R" with no named package, and the replication capsule was not consulted (paper-sourced only). Validated by (a) a **numpy oracle** re-implementing `S_q` (`q=1,2,∞`), the moving-block cyclic-shift set, i.i.d. identity-inclusion, and the eq-2 floor on hand-built residuals; (b) a **brute-force permutation-equivalence** test (independently materialising every shift and recomputing `S_q`) matching the production p-value bit-for-bit; (c) `_cwz_proxy_fit` vs `scipy` SLSQP on the simplex-LS; and (d) a **coverage simulation** (the `1−α` average-effect CI covers a known constant effect at ≈ `1−α`). -**Reference implementation:** authors' `Synth` package for R/MATLAB/Stata (`Synth::synth`); in-space placebo construction follows `SCtools::generate.placebos`. **R-parity anchor:** the Basque Country study (Abadie-Gardeazabal 2003, `data("basque")`) — published synthetic = region 10 (Cataluña) 0.851 + region 14 (Madrid) 0.149, `loss.v` 0.0089. Two-tier test (`tests/test_methodology_synthetic_control.py`): Tier-1 feeds R's `solution.v` via `custom_v` → donor weights match to atol 1e-3 (deterministic); Tier-2 checks the nested fit in a band. +**Reference implementation:** authors' `Synth` package for R/MATLAB/Stata (`Synth::synth`); in-space placebo construction follows ADH 2010 §3.4, not `SCtools::generate.placebos` (see the donor-pool Deviation above). **R-parity anchor:** the Basque Country study (Abadie-Gardeazabal 2003, `data("basque")`) — published synthetic = region 10 (Cataluña) 0.851 + region 14 (Madrid) 0.149, `loss.v` 0.0089. Two-tier test (`tests/test_methodology_synthetic_control.py`): Tier-1 feeds R's `solution.v` via `custom_v` → donor weights match to atol 1e-3 (deterministic); Tier-2 checks the nested fit in a band. **Requirements checklist:** - [x] Donor weights on the unit simplex; exactly one treated unit, block assignment. @@ -3888,7 +3889,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - [x] Out-of-sample cross-validation `V`-selection (`v_method="cv"`, ADH 2015 §; Abadie 2021 Eq. 9 `t0=T0/2`): per-window re-aggregation train/validation split (fully-spanning precondition) + deterministic flat-MSPE tie-break (fn. 7); threaded through the placebo refits. - [x] Inverse-variance `V` (`v_method="inverse_variance"`, Abadie 2021 §3.2(a)): closed-form `1/Var(X)`, zero-variance-row + all-zero-uniform-fallback handling. - [x] Gap path + pre-period RMSPE + predictor-balance table. -- [x] No analytical SE (NaN inference); in-space placebo permutation inference (`in_space_placebo()`, `rank/(n_placebos+1)`) with the real treated unit excluded from every placebo pool, effective-count denominator, and a scale-aware RMSPE-ratio floor. +- [x] No analytical SE (NaN inference); in-space placebo permutation inference (`in_space_placebo()`, `rank/(n_placebos+1)`) with the real treated unit in every placebo pool (ADH 2010 §3.4; an exact permutation), effective-count denominator, and a scale-aware RMSPE-ratio floor. - [x] Leave-one-out donor robustness (`leave_one_out()`, ADH 2015 §4): per-drop ATT / `delta_att` table + overlay gaps; fail-closed. - [x] In-time (backdating) placebo (`in_time_placebo()`, ADH 2015 §4): TRUNCATE windowing (drop held-out-window predictors + lockstep `custom_v` subset), feasible-date sweep, fail-closed. - [x] Confidence sets by test inversion (`test_sharp_null()` + `confidence_set()`, Firpo & Possebom 2018 §4): sharp-null `RMSPE^f` re-ranking of the in-space placebo gaps (Eqs 12–13) + constant/linear one-parameter sets (Eqs 14/16/18) with the strict `p^f > γ` boundary, EXACT piecewise-constant breakpoint inversion (no shape assumption; isolated/disjoint/unbounded sets handled), and fail-closed unbounded/empty/non-contiguous handling. *Deferred:* sensitivity weights (φ≠0), the general-θ menu (Eq 19), one-sided (§7), multiple-outcome/treated (§6). diff --git a/docs/methodology/papers/firpo-possebom-2018-review.md b/docs/methodology/papers/firpo-possebom-2018-review.md index cd34181ce..98e72b61f 100644 --- a/docs/methodology/papers/firpo-possebom-2018-review.md +++ b/docs/methodology/papers/firpo-possebom-2018-review.md @@ -150,7 +150,7 @@ A re-analysis of ETA terrorism on Basque Country GDP per capita (Abadie & Gardea ### Computational Considerations - The benchmark test (Eq. 5) is `O(J)` synthetic-control refits (the permutation reference set). The sensitivity analysis (Eqs. 8–9) is a **closed-form re-weighting** of the *already-computed* `RMSPE_j` plus a one-dimensional root-find for `φ̲`/`φ̄` — no refits. -- **Test-inversion CI = a grid search × the permutation test.** For each grid value `c` (or `c̃`): subtract `f(t)` from the relevant post-period outcomes, recompute `RMSPE^f` for all `J+1` units (Eq. 12), and evaluate Eq. 13. Because the donor synthetic controls and the pre-period denominators are unchanged across the grid (only the post-period gap shifts by `f(t)`), the per-grid-value cost is dominated by re-ranking, not refitting. Cost scales with grid resolution × `J`. +- **Test-inversion CI = a grid search × the permutation test.** For each grid value `c` (or `c̃`): subtract `f(t)` from the relevant post-period outcomes, recompute `RMSPE^f` for all `J+1` units (Eq. 12), and evaluate Eq. 13. Because the donor weights and the pre-period denominators are unchanged across the grid (only the post-period residuals move: the treated unit's by `−f(t)`, placebo `j`'s by `−(1 − w_j1)·f(t)`, where `w_j1` is its weight on the treated unit, which stays in the placebo pools, fn. 10), the per-grid-value cost is dominated by re-ranking, not refitting. Cost scales with grid resolution × `J`. - The general `ℝ^T` confidence set (Eq. 14) is computationally infeasible — an implementation must restrict to the constant / linear (or a small parametric) family and choose a finite grid. ### Tuning Parameters diff --git a/tests/test_methodology_synthetic_control.py b/tests/test_methodology_synthetic_control.py index c30732b79..a8fd13ff4 100644 --- a/tests/test_methodology_synthetic_control.py +++ b/tests/test_methodology_synthetic_control.py @@ -933,18 +933,94 @@ def test_in_space_placebo_strong_effect_ranks_treated_first(): assert set(pdf.loc[~pdf["is_treated"], "unit"]) == {"d0", "d1", "d2", "d3"} -def test_in_space_placebo_excludes_real_treated_from_donor_pools(): - # The real treated unit is never in the donor universe, so it cannot serve as - # a donor for any placebo (ADH 2010 contamination guard; SCtools convention). +def test_in_space_placebo_puts_real_treated_in_every_donor_pool(monkeypatch): + # ADH 2010 §3.4 (journal p. 501): each placebo run reassigns the intervention to a + # control unit, "shifting California to the donor pool", so every placebo is fit + # against all the other units, the real treated unit included. + import importlib + + sc = importlib.import_module("diff_diff.synthetic_control") res = _fit_for_placebo(n_donors=4) snap = res._fit_snapshot assert snap.treated_id == "treated" and "treated" not in snap.donor_ids + real_fit_unit = sc._placebo_fit_unit + pools = {} + + def recording_fit_unit(snap_arg, unit, donor_pool, n_starts): + pools[unit] = list(donor_pool) + return real_fit_unit(snap_arg, unit, donor_pool, n_starts) + + monkeypatch.setattr(sc, "_placebo_fit_unit", recording_fit_unit) with warnings.catch_warnings(): warnings.simplefilter("ignore") res.in_space_placebo() # Each donor became a placebo exactly once; the treated unit is not a placebo. assert "treated" not in res._placebo_gaps assert set(res._placebo_gaps) == set(snap.donor_ids) + units = ["d0", "d1", "d2", "d3", "treated"] + assert pools == {j: [u for u in units if u != j] for j in snap.donor_ids} + # Each placebo's weight on the treated unit is kept for the sharp-null re-ranking. + assert set(res._placebo_treated_weights) == set(snap.donor_ids) + + +def _factor_panel(seed=0, n_units=10, n_pre=24, n_post=6): + """No-effect panel: three AR(1) factors, U(0, 1) loadings, AR(1) noise, unit levels.""" + rng = np.random.default_rng(seed) + periods = n_pre + n_post + factors = np.empty((periods, 3)) + errors = np.empty((n_units, periods)) + factors[0] = rng.normal(size=3) / np.sqrt(0.75) + errors[:, 0] = rng.normal(size=n_units) / np.sqrt(0.75) + for t in range(1, periods): + factors[t] = 0.5 * factors[t - 1] + rng.normal(size=3) + errors[:, t] = 0.5 * errors[:, t - 1] + rng.normal(size=n_units) + loadings = rng.uniform(0.0, 1.0, (n_units, 3)) + return loadings @ factors.T + 0.5 * errors + rng.normal(size=(n_units, 1)) + + +def test_in_space_placebo_is_exact_permutation(): + # Under the null of no effect, moving the treatment label from unit to unit must only + # permute the J+1 RMSPE ratios. With the treated unit in every placebo's donor pool, + # each placebo refit is the fit that treats that unit, so the J+1 assignments give + # p = 1/(J+1), 2/(J+1), ..., 1 once each (no ties on this panel). + n_units, n_pre = 10, 24 + y = _factor_panel(n_units=n_units, n_pre=n_pre) + n_periods = y.shape[1] + fits = [] + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # poor-pre-fit notices on some units + for treated in range(n_units): + d = np.zeros(y.shape, dtype=int) + d[treated, n_pre:] = 1 + df = pd.DataFrame( + { + "unit": np.repeat(np.arange(n_units), n_periods), + "time": np.tile(np.arange(n_periods), n_units), + "y": y.ravel(), + "d": d.ravel(), + } + ) + res = SyntheticControl( + v_method="custom", + custom_v=np.ones(n_pre), + standardize="none", + inner_min_decrease=1e-3, + ).fit(df, "y", "d", "unit", "time", pre_period_outcomes="all") + res.in_space_placebo() + fits.append(res) + assert all(r.n_failed == 0 and r.n_placebos == n_units - 1 for r in fits) + p = np.array([r.placebo_p_value for r in fits]) + ratios = np.array([r.rmspe_ratio for r in fits]) + np.testing.assert_allclose(np.sort(p), np.arange(1, n_units + 1) / n_units) + np.testing.assert_allclose(p, [np.mean(ratios >= ratios[u]) for u in range(n_units)]) + for r in fits: + pdf = r.get_placebo_df() + placebos = pdf[~pdf["is_treated"]] + np.testing.assert_allclose( + placebos["rmspe_ratio"].to_numpy(dtype=float), + ratios[placebos["unit"].to_numpy(dtype=int)], + rtol=1e-12, + ) def test_in_space_placebo_p_in_valid_discrete_set(): @@ -1922,7 +1998,7 @@ def test_leave_one_out_uniform_shift_surfaced_by_delta_not_range(monkeypatch): def uniform_shift(snap_arg, unit, pool, n_starts): gp = {p: 0.0 for p in snap.pre_periods} gp.update({p: baseline + shift for p in snap.post_periods}) - return (gp, 1.0), "ran" + return (gp, 1.0, {}), "ran" monkeypatch.setattr(sc, "_placebo_fit_unit", uniform_shift) with warnings.catch_warnings(): @@ -3111,9 +3187,10 @@ def test_cv_in_time_placebo_flat_window_is_infeasible_not_failed(): def test_cv_in_space_placebo_excludes_donor_flat_refits(): # In-space placebo path (solve-level sentinel): the FULL donor set is distinguishable so - # the headline fit is well-posed, but dropping donor d0 (pseudo-treating it) leaves the - # remaining donors {d1,d2,d3} identical in the validation window -> that placebo's pool - # is donor-indistinguishable and must be EXCLUDED (not enter with arbitrary "converged" + # the headline fit is well-posed, but pseudo-treating donor d0 leaves its pool + # {d1,d2,d3,treated} identical in the validation window (the real treated unit is in + # every placebo's pool and equals d1..d3 there) -> that placebo's pool is + # donor-indistinguishable and must be EXCLUDED (not enter with arbitrary "converged" # weights). Placebos for d1/d2/d3 keep d0 in the pool, so they remain identified. rng = np.random.default_rng(3) years = list(range(2000, 2008)) @@ -3128,12 +3205,12 @@ def test_cv_in_space_placebo_excludes_donor_flat_refits(): y = 10.0 + rng.normal(0, 0.2) rows.append({"unit": f"d{j}", "year": yr, "y": y, "treated": 0}) for yr in years: - y = 12.0 if yr in (2003, 2004, 2005) else (5.5 + rng.normal(0, 0.2) if yr <= 2005 else 13.0) + y = 10.0 if yr in (2003, 2004, 2005) else (5.5 + rng.normal(0, 0.2) if yr <= 2005 else 13.0) rows.append({"unit": "treated", "year": yr, "y": y, "treated": int(yr >= 2006)}) df = pd.DataFrame(rows) res = _fit_cv(df, seed=0) assert np.isfinite(res.att) # headline well-posed: the full donor set is distinguishable - # The d0 placebo (pool {d1,d2,d3} identical in val) is dropped as STRUCTURALLY + # The d0 placebo (pool {d1,d2,d3,treated} identical in val) is dropped as STRUCTURALLY # infeasible (cv donor-indistinguishability), NOT a solver "failed": the threading # from _outer_solve_V_cv's structural sentinel through _placebo_fit_unit routes it to # n_infeasible. The others keep d0 in the pool, so they remain identified and enter. @@ -3287,6 +3364,32 @@ def test_sharp_null_pvalue_matches_independent_oracle(): assert p == pytest.approx(p_o), (c, p, p_o) +# Weight each oracle placebo puts on the treated unit (in every placebo's donor pool). +_ORACLE_TREATED_WEIGHTS = {"a": 0.5, "b": 0.0, "c": 0.25, "d": 1.0} + + +def test_sharp_null_pvalue_imputes_treated_in_placebo_pools(): + # Under H_0^f the treated unit's untreated outcome is Y_1 - f, so placebo j's synthetic + # control moves by -w_j1 * f: its post gap moves by +w_j1 * f before Eq 12 subtracts f. + pre_denoms = _oracle_pre_denoms() + w1 = _ORACLE_TREATED_WEIGHTS + for c in (0.0, 1.0, 1.65, 2.0, 2.5, 3.5, -1.0): + f = _constant_f_post(c, len(_ORACLE_POST)) + p, r1, n_ref = _sharp_null_pvalue( + _ORACLE_TREATED, _ORACLE_PLACEBOS, _ORACLE_POST, f, pre_denoms, 0, treated_weights=w1 + ) + resid_t = np.array([_ORACLE_TREATED[q] for q in _ORACLE_POST]) - f + r1_o = float(np.sqrt(np.mean(resid_t**2) / pre_denoms[0])) + rj = [] + for u, gp in _ORACLE_PLACEBOS.items(): + null_gap = np.array([gp[q] for q in _ORACLE_POST]) + w1[u] * f + rj.append(float(np.sqrt(np.mean((null_gap - f) ** 2) / pre_denoms[u]))) + p_o = (1 + sum(1 for r in rj if r >= r1_o)) / (len(rj) + 1) + assert n_ref == 4 + assert r1 == pytest.approx(r1_o) + assert p == pytest.approx(p_o), (c, p, p_o) + + def test_sharp_null_zero_equals_rmspe_ratio_helper(): # Eq 13 at f==0 reduces to the ADH RMSPE-ratio statistic (Eq 4). scale = 1.0 @@ -3479,6 +3582,31 @@ def width(g): assert w_lo >= w_hi - 1e-9 +def test_invert_with_treated_weights_matches_direct_pvalues(): + # Placebo j's residual is g_j - (1 - w_j1) * c, so its breakpoint quadratic carries the + # (1 - w_j1)**2 * S2 curvature. The grid p-values must equal _sharp_null_pvalue's, and the + # exact set's endpoints must be where that p-value crosses gamma. + pre_denoms = _oracle_pre_denoms() + w1 = _ORACLE_TREATED_WEIGHTS + gamma = 0.45 + args = (_ORACLE_TREATED, _ORACLE_PLACEBOS, _ORACLE_POST, pre_denoms, 0, "constant", gamma) + + def p_direct(c): + f = _constant_f_post(c, len(_ORACLE_POST)) + return _sharp_null_pvalue( + _ORACLE_TREATED, _ORACLE_PLACEBOS, _ORACLE_POST, f, pre_denoms, 0, treated_weights=w1 + )[0] + + grid = _invert_sharp_null(*args, bounds=(-6.0, 10.0), n_grid=321, treated_weights=w1)["grid"] + for c, p, _ in grid: + assert p == pytest.approx(p_direct(c)), c + exact = _invert_sharp_null(*args, treated_weights=w1) + assert exact["status"] == "ran" and exact["contiguous"] + lo, hi = exact["lower"], exact["upper"] + assert p_direct(lo - 1e-6) <= gamma < p_direct(lo + 1e-6) + assert p_direct(hi + 1e-6) <= gamma < p_direct(hi - 1e-6) + + # --- end-to-end (real fit): custom V skips the outer search for speed/determinism --- @@ -3539,6 +3667,44 @@ def test_test_sharp_null_zero_equals_placebo_p_value_end_to_end(): assert s0["n_placebos"] == res.n_placebos +def test_test_sharp_null_matches_refits_on_null_imputed_panel(): + # Brute force for H_0: alpha_1t = c. Replace the treated unit's post outcomes by its + # untreated outcomes Y_1t - c, refit every placebo from scratch on that panel, and rank + # Eq 12's statistic. The weights depend on pre-period data only, so each refit's post gap + # is the stored placebo gap plus w_j1 * c: test_sharp_null needs no refits. + res, years, T0 = _fit_with_placebos() + df, _, _ = _make_panel(n_donors=6, T=10, T0=T0, effect=3.0, seed=0) + post = years[T0:] + assert max(res._placebo_treated_weights.values()) > 0.1 # the shift is exercised + for c in (1.5, 3.75): # at 3.75 the p-value differs if the w_j1 * c shift is dropped + f = _constant_f_post(c, len(post)) + null_df = df.copy() + null_df.loc[(null_df["unit"] == "treated") & (null_df["year"] >= post[0]), "y"] -= c + r1 = _rmspe_f_ratio(res.gap_path, post, f, res._placebo_pre_denoms["treated"]) + n_ge = 0 + for j, stored in res._placebo_gaps.items(): + treated_j = ((null_df["unit"] == j) & (null_df["year"] >= post[0])).astype(int) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + refit = SyntheticControl(v_method="custom", custom_v=np.ones(T0), seed=0).fit( + null_df.assign(treated=treated_j), + "y", + "treated", + "unit", + "year", + post_periods=post, + treated_unit=j, + ) + w_j1 = res._placebo_treated_weights[j] + for t in post: + assert refit.gap_path[t] == pytest.approx(stored[t] + w_j1 * c, abs=1e-9) + n_ge += _rmspe_f_ratio(refit.gap_path, post, f, res._placebo_pre_denoms[j]) >= r1 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + s = res.test_sharp_null(c) + assert s["p_value"] == pytest.approx((1 + n_ge) / (res.n_placebos + 1)) + + def test_confidence_set_constant_contains_att_and_excludes_zero(): res = _exact_combo_fit(effect=3.0) with warnings.catch_warnings(): From 9caffeb365d8c8245fc11d0936b5e2de83b64b00 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 14 Oct 2026 16:30:56 -0400 Subject: [PATCH 2/5] docs(synthetic_control): record the Firpo-Possebom fn 10 reading of ADH and qualify placebo exactness Firpo & Possebom (2018) footnote 10 characterizes ADH 2010/2015 as excluding the treated unit from the placebo pools, which is at odds with the ADH 2010 sentence the donor-pool Note quotes; record both readings and which one the implementation follows. State that the permutation p-value is exact only when every placebo refit enters the reference set (failed / infeasible refits are excluded and shrink the effective denominator), in the REGISTRY Note, the in_space_placebo() docstring and the changelog fragment. --- changelog.d/20261005-in-space-placebo-donor-pool.md | 4 +++- diff_diff/synthetic_control_results.py | 10 +++++++--- docs/methodology/REGISTRY.md | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/changelog.d/20261005-in-space-placebo-donor-pool.md b/changelog.d/20261005-in-space-placebo-donor-pool.md index 415c29488..e3cf707a2 100644 --- a/changelog.d/20261005-in-space-placebo-donor-pool.md +++ b/changelog.d/20261005-in-space-placebo-donor-pool.md @@ -5,7 +5,9 @@ p. 501) do ("shifting California to the donor pool"). It used to drop the treated unit from every placebo's donor pool (the `SCtools::generate.placebos` construction); that reference set changes with the treated unit, so the - permutation p-value was not exact. Every placebo that puts weight on the + permutation p-value was not exact. (The new p-value is exact when every + placebo refit enters the reference set; failed or infeasible refits are still + excluded and reported via `n_failed` / `n_infeasible`.) Every placebo that puts weight on the treated unit now gets a different RMSPE ratio, so `placebo_p_value` and the placebo rows of `get_placebo_df()` change on every such fit, by whole ranks; the fit itself (`att`, weights, the treated unit's `rmspe_ratio`) does not diff --git a/diff_diff/synthetic_control_results.py b/diff_diff/synthetic_control_results.py index 7748a74aa..c232073a0 100644 --- a/diff_diff/synthetic_control_results.py +++ b/diff_diff/synthetic_control_results.py @@ -880,9 +880,13 @@ def in_space_placebo( intervention to a control state, "shifting California to the donor pool"). All ``J+1`` statistics then come from one construction that does not depend on which unit is treated, so under the null of no effect (and random - assignment) the permutation p-value is exact. (``SCtools::generate.placebos`` - instead drops the treated unit from every placebo's pool; that reference set - changes with the treated unit, so its p-value is not exact.) The ranking set + assignment) the permutation p-value is exact whenever every placebo refit + enters the reference set (a failed or infeasible refit is excluded and + shrinks the effective denominator; see ``n_failed`` / ``n_infeasible``). + (``SCtools::generate.placebos`` instead drops the treated unit from every + placebo's pool - the reading Firpo & Possebom's fn 10 attributes to ADH - + and that reference set changes with the treated unit, so its p-value is + not exact.) The ranking set is the ``J+1`` units ``{treated} ∪ {J placebos}``. The post/pre RMSPE ratio normalizes by pre-treatment fit, which obviates the pre-fit-cutoff filtering of ADH Figures 5-7 (journal p. 502), so no pre-fit filter is diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index a5f61deba..84338b9b3 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -3858,7 +3858,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Deviation from R:** `standardize="none"` disables predictor standardization entirely; R `Synth` always scales by the predictor SD. Provided for diagnostics; changes the geometry of the `V` objective. - **Note:** predictor rows support only **equal-weight** linear combinations of pre-period values — `mean` (`k_s = 1/T0`), `sum` (`k_s = 1`), and per-period outcome lags (identity, a single `k_s = 1`). ADH (2010) §2.3 defines the general form `Ȳ_i^{K_m} = Σ_s k_s Y_is` with *arbitrary* weights `k_s`; this release does NOT accept user-supplied non-uniform `K_m` weight vectors (and `median` and other non-linear aggregations are intentionally excluded). The supported set still spans the standard `Synth::dataprep` `predictors.op` + `special.predictors` usage; arbitrary-weight `K_m` is a deferred extension. - **Deviation from R:** predictor/outcome **aggregation fails closed on any non-finite (NaN/inf) cell**, whereas R `Synth::dataprep` hardwires `na.rm=TRUE` (aggregating over the observed cells of a partially-missing window). The fail-closed contract is deliberate: na-dropping silently aggregates different period subsets across units, yielding incomparable predictors with no warning. The analyst must restrict `predictor_window` / `special_predictors` / `pre_period_outcomes` periods (and the outcome panel) to where each variable is observed; both partially- and fully-missing windows raise `ValueError`. Only the row *ordering* matches `dataprep`, not the missing-data handling. -- **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015). The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, so with the default donor pool it reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. +- **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015), "because, under sharp null hypotheses, the treatment effect is known". (That same footnote, relaying an anonymous referee, characterizes ADH 2010 and ADH 2015 as *excluding* the treated unit from the placebo pools "to avoid bias when there is a treatment effect" - a reading at odds with the ADH 2010 sentence quoted above. This implementation follows ADH 2010's printed construction and the Firpo & Possebom / Imbens & Rubin convention, which coincide; the exclusion reading is the `SCtools` construction recorded as the Deviation below.) The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, so with the default donor pool it reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Exactness is conditional on every placebo refit entering the reference set: a failed or structurally infeasible refit is excluded and shrinks the effective denominator (see the failure-handling Note below), which the runtime warning surfaces. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. - **Deviation from R (`SCtools::generate.placebos`):** SCtools drops the treated unit from every placebo's pool, fitting each placebo against the other `J−1` donors. That reference set changes with the treated unit, so its p-value is not exact: on the 10-unit no-effect panel of `test_in_space_placebo_is_exact_permutation`, leaving the treated unit out of the placebo pools gives `p ≤ 0.2` for 3 of the 10 assignments. Every placebo that puts weight on the treated unit has a different RMSPE ratio here than in SCtools, so `placebo_p_value` can differ by whole ranks. - **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — raise `n_starts` on `in_space_placebo()` or re-fit with a larger `optimizer_options['maxiter']`). - **Note (in-space / leave-one-out infeasible vs failed split):** an excluded `in_space_placebo()` / `leave_one_out()` refit is attributed to one of two causes, mirroring the split `in_time_placebo` already reports. A **solver non-convergence** (a truncated inner `W` or outer `V` search) is tallied in `n_failed` (`_loo_n_failed`) with a `status="failed"` row; a **structural cv infeasibility** — under `v_method="cv"`, the pseudo-treated donor pool (in-space) or reduced pool (leave-one-out) is indistinguishable in a re-aggregated CV window, so the weights are unidentified (the window-information gate above; `_outer_solve_V_cv` returns a structural sentinel that `_placebo_fit_unit` threads out) — is tallied in `n_infeasible` (`_loo_n_infeasible`) with a `status="infeasible"` row. Both are excluded from the rank / ATT range identically, so `placebo_p_value` / `n_placebos` are **unchanged** by the attribution — only the diagnostic accounting is refined. `_placebo_status` / `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. `all_refits_infeasible` / `all_refits_unusable`) codes for a no-usable-refit run that is purely structural or a mix, and `DiagnosticReport` surfaces the machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. `n_infeasible` is 0 for the non-`cv` `v_method`s (no structural-identification gate). The remedy differs by cause: a structural infeasibility needs different predictors / `v_cv_t0` / donor pool, NOT a larger `n_starts` / `inner_max_iter`. From c46c9499de212598422be3453deb28f98ce1ddbf Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 14 Oct 2026 17:47:58 -0400 Subject: [PATCH 3/5] fix(synthetic_control): canonicalize explicit donor_pool order so the treated fit matches its placebo refits fit() now orders an explicit donor_pool by the pivot's sorted unit order, the same assignment-independent order every in-space placebo refit uses. Frank-Wolfe breaks exact gradient ties by position, so a caller-ordered list could give the treated fit a different tie-break than its placebos and make the permutation rank depend on list presentation. Regression test with two pre-period-identical donors and three list orders (same weights, ATT, rmspe_ratio, placebo p-value and table). Also qualify the exactness claim in llms-full.txt the same way as the docstring, REGISTRY and the changelog fragment. --- .../20261005-in-space-placebo-donor-pool.md | 5 +++ diff_diff/guides/llms-full.txt | 2 +- diff_diff/synthetic_control.py | 12 +++++- docs/methodology/REGISTRY.md | 2 +- tests/test_methodology_synthetic_control.py | 38 +++++++++++++++++++ 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/changelog.d/20261005-in-space-placebo-donor-pool.md b/changelog.d/20261005-in-space-placebo-donor-pool.md index e3cf707a2..b0acc6c80 100644 --- a/changelog.d/20261005-in-space-placebo-donor-pool.md +++ b/changelog.d/20261005-in-space-placebo-donor-pool.md @@ -23,3 +23,8 @@ change too. With exactly 2 donors each placebo is now fit against 2 units (a real weight solve and `V` search instead of a forced `w = [1]`); fewer than 2 donors still gives a NaN p-value. + `fit()` now processes an explicit `donor_pool` in the panel's sorted unit + order (the order the placebo refits use) rather than the caller's list order, + so `donor_weights` / `get_weights_df()` rows follow that order and an exact + weight tie between donors resolves the same way in the treated fit and in + every placebo. diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index af347c68e..292693505 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -675,7 +675,7 @@ scm.fit( ) -> SyntheticControlResults ``` -**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits it against all the other units (the real treated unit is in every placebo pool, ADH 2010 §3.4, so the permutation is exact), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). +**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits it against all the other units (the real treated unit is in every placebo pool, ADH 2010 §3.4, so the permutation is exact under the null whenever every placebo refit enters the reference set; failed / infeasible refits are excluded and shrink the effective denominator, see `n_failed` / `n_infeasible`), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). **Usage:** diff --git a/diff_diff/synthetic_control.py b/diff_diff/synthetic_control.py index 642e64001..f890a7a95 100644 --- a/diff_diff/synthetic_control.py +++ b/diff_diff/synthetic_control.py @@ -291,7 +291,10 @@ def fit( to all pre-period outcomes. donor_pool : list, optional Explicit donor unit identifiers (must be never-treated). Defaults to - all never-treated units. + all never-treated units. The list's order is not used: donors are + processed in the panel's sorted unit order (the order the in-space + placebo refits also use), so the fit does not depend on how the list + was written. survey_design : optional Not yet supported — raises ``NotImplementedError`` if provided. @@ -369,6 +372,13 @@ def fit( for v in needed_vars } Y = pivots[outcome] + # Canonicalize the donor order to the pivot's sorted unit order, so an explicit + # ``donor_pool`` list fits the same way whichever order the caller wrote it in, + # and the treated fit uses the SAME assignment-independent ordering as every + # in-space placebo refit (whose pool is "all other units" in pivot order). The + # Frank-Wolfe weight solve breaks exact gradient ties by position, so without + # this the rank could depend on list presentation rather than on assignment. + donor_ids = [u for u in Y.columns if u != treated_id] Z1 = Y.loc[pre_periods, treated_id].to_numpy(dtype=float) Z0 = Y.loc[pre_periods, donor_ids].to_numpy(dtype=float) # (n_pre, J) diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 84338b9b3..f64d03025 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -3858,7 +3858,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Deviation from R:** `standardize="none"` disables predictor standardization entirely; R `Synth` always scales by the predictor SD. Provided for diagnostics; changes the geometry of the `V` objective. - **Note:** predictor rows support only **equal-weight** linear combinations of pre-period values — `mean` (`k_s = 1/T0`), `sum` (`k_s = 1`), and per-period outcome lags (identity, a single `k_s = 1`). ADH (2010) §2.3 defines the general form `Ȳ_i^{K_m} = Σ_s k_s Y_is` with *arbitrary* weights `k_s`; this release does NOT accept user-supplied non-uniform `K_m` weight vectors (and `median` and other non-linear aggregations are intentionally excluded). The supported set still spans the standard `Synth::dataprep` `predictors.op` + `special.predictors` usage; arbitrary-weight `K_m` is a deferred extension. - **Deviation from R:** predictor/outcome **aggregation fails closed on any non-finite (NaN/inf) cell**, whereas R `Synth::dataprep` hardwires `na.rm=TRUE` (aggregating over the observed cells of a partially-missing window). The fail-closed contract is deliberate: na-dropping silently aggregates different period subsets across units, yielding incomparable predictors with no warning. The analyst must restrict `predictor_window` / `special_predictors` / `pre_period_outcomes` periods (and the outcome panel) to where each variable is observed; both partially- and fully-missing windows raise `ValueError`. Only the row *ordering* matches `dataprep`, not the missing-data handling. -- **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015), "because, under sharp null hypotheses, the treatment effect is known". (That same footnote, relaying an anonymous referee, characterizes ADH 2010 and ADH 2015 as *excluding* the treated unit from the placebo pools "to avoid bias when there is a treatment effect" - a reading at odds with the ADH 2010 sentence quoted above. This implementation follows ADH 2010's printed construction and the Firpo & Possebom / Imbens & Rubin convention, which coincide; the exclusion reading is the `SCtools` construction recorded as the Deviation below.) The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, so with the default donor pool it reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Exactness is conditional on every placebo refit entering the reference set: a failed or structurally infeasible refit is excluded and shrinks the effective denominator (see the failure-handling Note below), which the runtime warning surfaces. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. +- **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015), "because, under sharp null hypotheses, the treatment effect is known". (That same footnote, relaying an anonymous referee, characterizes ADH 2010 and ADH 2015 as *excluding* the treated unit from the placebo pools "to avoid bias when there is a treatment effect" - a reading at odds with the ADH 2010 sentence quoted above. This implementation follows ADH 2010's printed construction and the Firpo & Possebom / Imbens & Rubin convention, which coincide; the exclusion reading is the `SCtools` construction recorded as the Deviation below.) The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, and `fit()` canonicalizes an explicit `donor_pool` to that same order - the Frank-Wolfe weight solve breaks exact gradient ties by position, so a caller-ordered list could otherwise give the treated fit a different tie-break than its placebos - so every refit reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Exactness is conditional on every placebo refit entering the reference set: a failed or structurally infeasible refit is excluded and shrinks the effective denominator (see the failure-handling Note below), which the runtime warning surfaces. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. - **Deviation from R (`SCtools::generate.placebos`):** SCtools drops the treated unit from every placebo's pool, fitting each placebo against the other `J−1` donors. That reference set changes with the treated unit, so its p-value is not exact: on the 10-unit no-effect panel of `test_in_space_placebo_is_exact_permutation`, leaving the treated unit out of the placebo pools gives `p ≤ 0.2` for 3 of the 10 assignments. Every placebo that puts weight on the treated unit has a different RMSPE ratio here than in SCtools, so `placebo_p_value` can differ by whole ranks. - **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — raise `n_starts` on `in_space_placebo()` or re-fit with a larger `optimizer_options['maxiter']`). - **Note (in-space / leave-one-out infeasible vs failed split):** an excluded `in_space_placebo()` / `leave_one_out()` refit is attributed to one of two causes, mirroring the split `in_time_placebo` already reports. A **solver non-convergence** (a truncated inner `W` or outer `V` search) is tallied in `n_failed` (`_loo_n_failed`) with a `status="failed"` row; a **structural cv infeasibility** — under `v_method="cv"`, the pseudo-treated donor pool (in-space) or reduced pool (leave-one-out) is indistinguishable in a re-aggregated CV window, so the weights are unidentified (the window-information gate above; `_outer_solve_V_cv` returns a structural sentinel that `_placebo_fit_unit` threads out) — is tallied in `n_infeasible` (`_loo_n_infeasible`) with a `status="infeasible"` row. Both are excluded from the rank / ATT range identically, so `placebo_p_value` / `n_placebos` are **unchanged** by the attribution — only the diagnostic accounting is refined. `_placebo_status` / `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. `all_refits_infeasible` / `all_refits_unusable`) codes for a no-usable-refit run that is purely structural or a mix, and `DiagnosticReport` surfaces the machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. `n_infeasible` is 0 for the non-`cv` `v_method`s (no structural-identification gate). The remedy differs by cause: a structural infeasibility needs different predictors / `v_cv_t0` / donor pool, NOT a larger `n_starts` / `inner_max_iter`. diff --git a/tests/test_methodology_synthetic_control.py b/tests/test_methodology_synthetic_control.py index a8fd13ff4..148b7ad6e 100644 --- a/tests/test_methodology_synthetic_control.py +++ b/tests/test_methodology_synthetic_control.py @@ -281,6 +281,44 @@ def test_donor_pool_restricts_donors(): assert set(res.get_weights_df()["unit"]) <= {"d0", "d1"} +def test_explicit_donor_pool_order_does_not_change_fit_or_placebo_rank(): + # Two donors (d0, d1) are identical in the pre-period and differ post, and the treated + # unit's pre path equals theirs, so the simplex weight solve faces an exact gradient + # tie that Frank-Wolfe breaks by position. fit() canonicalizes an explicit donor_pool + # to the pivot's sorted order (the order every in-space placebo refit uses), so the + # weights, the ATT and the placebo rank cannot depend on how the list was written. + years = list(range(2000, 2010)) + T0 = 7 + pre = 10.0 + 0.5 * np.arange(T0) + rows = [] + for name, post in (("d0", [14.0, 14.5, 15.0]), ("d1", [11.0, 11.0, 11.0])): + for yr, y in zip(years, list(pre) + post): + rows.append({"unit": name, "year": yr, "y": y, "treated": 0}) + for yr, y in zip(years, list(5.0 + 0.2 * np.arange(T0)) + [6.0, 6.1, 6.2]): + rows.append({"unit": "d2", "year": yr, "y": y, "treated": 0}) + for yr, y in zip(years, list(pre) + [16.0, 16.5, 17.0]): + rows.append({"unit": "treated", "year": yr, "y": y, "treated": int(yr >= years[T0])}) + df = pd.DataFrame(rows) + fits = [] + for pool in (["d0", "d1", "d2"], ["d1", "d0", "d2"], ["d2", "d1", "d0"]): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res = SyntheticControl(v_method="custom", custom_v=np.ones(T0), seed=0).fit( + df, "y", "treated", "unit", "year", donor_pool=pool + ) + res.in_space_placebo() + fits.append(res) + base = fits[0] + assert list(base._fit_snapshot.donor_ids) == ["d0", "d1", "d2"] + for other in fits[1:]: + assert list(other._fit_snapshot.donor_ids) == ["d0", "d1", "d2"] + assert other.att == base.att + assert other.donor_weights == base.donor_weights + assert other.rmspe_ratio == base.rmspe_ratio + assert other.placebo_p_value == base.placebo_p_value + pd.testing.assert_frame_equal(other.get_placebo_df(), base.get_placebo_df()) + + def test_contaminated_donor_pool_rejected(): df, years, T0 = _make_panel() # The treated unit itself must never appear in the donor pool. From 030a55d28ed2183a75cd0e385a009869e9814307 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 14 Oct 2026 18:00:35 -0400 Subject: [PATCH 4/5] fix(synthetic_control): reject an in_space_placebo n_starts override that differs from the fit under a V search Under v_method="nested" / "cv" the multistart count is part of the fitting procedure: more starts can select a different V optimum even when every fit converges. The treated unit's RMSPE ratio was produced at fit time with the fit's n_starts, so letting the placebos use another budget ranks one statistic against references computed by a different rule and voids the exactness argument. in_space_placebo(n_starts=k) with k != the fit's n_starts now raises ValueError pointing at a re-fit; the lazy reference build behind test_sharp_null / confidence_set goes through the same gate. The value stays inert under custom / inverse_variance (no V search). Remedy texts that said "raise n_starts on in_space_placebo()" now say to re-fit. Tests for the gate, for the inert case, and the cache-invalidation test rebuilds with the fit's own budget. --- .../20261005-in-space-placebo-donor-pool.md | 11 ++++++ diff_diff/synthetic_control_results.py | 39 ++++++++++++++----- docs/methodology/REGISTRY.md | 2 +- tests/test_methodology_synthetic_control.py | 34 +++++++++++++++- 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/changelog.d/20261005-in-space-placebo-donor-pool.md b/changelog.d/20261005-in-space-placebo-donor-pool.md index b0acc6c80..eacc2f0ee 100644 --- a/changelog.d/20261005-in-space-placebo-donor-pool.md +++ b/changelog.d/20261005-in-space-placebo-donor-pool.md @@ -28,3 +28,14 @@ so `donor_weights` / `get_weights_df()` rows follow that order and an exact weight tie between donors resolves the same way in the treated fit and in every placebo. + +### Behavioral Changes +- **`SyntheticControlResults.in_space_placebo(n_starts=...)` rejects a multistart + count that differs from the fit's under `v_method="nested"` / `"cv"`** (also + reached through the lazy reference build in `test_sharp_null()` / + `confidence_set()`). The treated unit's RMSPE ratio was produced at fit time + with the fit's `n_starts`, and more starts can select a different `V` optimum + even when every fit converges, so a different placebo budget would rank one + statistic against references computed by another procedure and void the + exactness of the permutation. Re-fit with the larger `n_starts` instead. The + override stays inert under `custom` / `inverse_variance` (no `V` search). diff --git a/diff_diff/synthetic_control_results.py b/diff_diff/synthetic_control_results.py index c232073a0..93c74a508 100644 --- a/diff_diff/synthetic_control_results.py +++ b/diff_diff/synthetic_control_results.py @@ -911,7 +911,7 @@ def in_space_placebo( **inherits the original fit's ``optimizer_options`` / ``n_starts``**, so valid inference requires settings adequate for the outer ``V`` search to converge: production defaults do; - with cheap settings, raise ``n_starts`` here or re-fit with a larger + with cheap settings, re-fit with a larger ``n_starts`` or ``optimizer_options['maxiter']`` (otherwise placebos are dropped as failed). The treated unit's own fit is held to the same standard — if its inner OR outer search did not converge, the whole run fails closed (see below). @@ -919,10 +919,14 @@ def in_space_placebo( Parameters ---------- n_starts : int, optional - Override the multistart count for each placebo's outer V search (nested/cv). - Default None inherits the original fit's ``n_starts``. The placebo - loop is the cost driver (one outer V search per donor); lower it for a - faster, coarser scan. + Multistart count for each placebo's outer V search. Default None inherits + the original fit's ``n_starts``. Under ``v_method="nested"`` / ``"cv"`` a + value different from the fit's is **rejected** (``ValueError``): the + treated unit's ratio was produced with the fit's ``n_starts``, and the + permutation is exact only when every one of the ``J+1`` statistics comes + from the same procedure. To change the budget, re-fit with the new + ``n_starts``. Under ``"custom"`` / ``"inverse_variance"`` there is no V + search and the value is inert. Returns ------- @@ -963,6 +967,23 @@ def in_space_placebo( if not isinstance(n_starts, (int, np.integer)) or n_starts < 1: raise ValueError(f"n_starts override must be a positive integer, got {n_starts!r}") n_starts_eff = int(n_starts) + # The permutation is exact only if every one of the J+1 statistics is the + # output of the SAME procedure. Under a nested / cv V search the multistart + # count is part of that procedure (more starts can land on a different V + # optimum even when every fit converges), and the treated unit's ratio was + # produced at fit time with snap.n_starts - so a different placebo budget + # would rank a treated statistic against references computed by another + # rule. Reject it; the remedy is one re-fit with the larger n_starts. + # custom / inverse_variance run no V search, so n_starts is inert there. + if snap.v_method in ("nested", "cv") and n_starts_eff != snap.n_starts: + raise ValueError( + f"in_space_placebo(n_starts={n_starts_eff}) differs from the fit's " + f"n_starts={snap.n_starts} under v_method={snap.v_method!r}. Every " + "placebo must be fit by the same procedure as the treated unit for " + "the permutation p-value to be exact, so the override is rejected. " + f"Re-fit with SyntheticControl(n_starts={n_starts_eff}) and call " + "in_space_placebo() without an override." + ) treated_pre = _mspe(self.gap_path, snap.pre_periods) treated_post = _mspe(self.gap_path, snap.post_periods) @@ -1122,8 +1143,8 @@ def in_space_placebo( warnings.warn( f"{n_failed} of {n_donors} in-space placebos failed to reach a valid " "optimum (a non-converged inner weight solve or outer V search) and were " - "excluded with status='failed'; raise n_starts or loosen the optimizer " - f"tolerances. placebo_p_value uses the remaining {n_placebos}.", + "excluded with status='failed'; re-fit with a larger n_starts or loosen the " + f"optimizer tolerances. placebo_p_value uses the remaining {n_placebos}.", UserWarning, stacklevel=2, ) @@ -2258,8 +2279,8 @@ def _require_placebo_reference(self, n_starts: Optional[int]) -> None: warnings.warn( "n_starts is ignored: the in-space placebo reference set was already " "computed and is reused (every sharp null / grid value re-ranks the same " - "placebo gaps). Re-run in_space_placebo(n_starts=...) explicitly to rebuild " - "it with a different multistart count.", + "placebo gaps). Re-fit with a different n_starts to rebuild it with another " + "multistart count (the placebos must use the fit's own n_starts).", UserWarning, stacklevel=3, ) diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index f64d03025..6241ffcbe 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -3860,7 +3860,7 @@ Classic synthetic control (donor/unit weights only) for a single treated unit, d - **Deviation from R:** predictor/outcome **aggregation fails closed on any non-finite (NaN/inf) cell**, whereas R `Synth::dataprep` hardwires `na.rm=TRUE` (aggregating over the observed cells of a partially-missing window). The fail-closed contract is deliberate: na-dropping silently aggregates different period subsets across units, yielding incomparable predictors with no warning. The analyst must restrict `predictor_window` / `special_predictors` / `pre_period_outcomes` periods (and the outcome panel) to where each variable is observed; both partially- and fully-missing windows raise `ValueError`. Only the row *ordering* matches `dataprep`, not the missing-data handling. - **Note (in-space placebo donor pool):** the real treated unit is **in every placebo's donor pool** — when donor `j` is pseudo-treated it is fit against the other `J` units (the remaining `J−1` donors plus the real treated unit), as the treated unit is fit against its `J` donors. ADH 2010 §3.4 (journal p. 501) states the construction: "In each iteration we reassign in our data the tobacco control intervention to one of the 38 control states, shifting California to the donor pool." Firpo & Possebom (2018, fn 10) keep the treated unit in the placebo pools as well, following Imbens & Rubin (2015), "because, under sharp null hypotheses, the treatment effect is known". (That same footnote, relaying an anonymous referee, characterizes ADH 2010 and ADH 2015 as *excluding* the treated unit from the placebo pools "to avoid bias when there is a treatment effect" - a reading at odds with the ADH 2010 sentence quoted above. This implementation follows ADH 2010's printed construction and the Firpo & Possebom / Imbens & Rubin convention, which coincide; the exclusion reading is the `SCtools` construction recorded as the Deviation below.) The symmetry is what makes the permutation test exact: each unit's RMSPE ratio comes from one construction that does not depend on which unit is treated (each refit takes its pool in the pivot's sorted unit order, and `fit()` canonicalizes an explicit `donor_pool` to that same order - the Frank-Wolfe weight solve breaks exact gradient ties by position, so a caller-ordered list could otherwise give the treated fit a different tie-break than its placebos - so every refit reproduces a fresh `fit()` that treats that unit), so under the null of no effect and random assignment the treated unit's rank is uniform over the `J+1` units. Exactness is conditional on every placebo refit entering the reference set: a failed or structurally infeasible refit is excluded and shrinks the effective denominator (see the failure-handling Note below), which the runtime warning surfaces. Under that null the treated unit's post-period outcome is its untreated outcome, so it is a valid donor; the sharp-null test inversion accounts for it when `f ≠ 0` (see above). The ranking set is the `J+1` units {treated} ∪ {J placebos}. `J < 2` still returns NaN: with one donor, the treated unit and the donor are each other's synthetic control. - **Deviation from R (`SCtools::generate.placebos`):** SCtools drops the treated unit from every placebo's pool, fitting each placebo against the other `J−1` donors. That reference set changes with the treated unit, so its p-value is not exact: on the 10-unit no-effect panel of `test_in_space_placebo_is_exact_permutation`, leaving the treated unit out of the placebo pools gives `p ≤ 0.2` for 3 of the 10 assignments. Every placebo that puts weight on the treated unit has a different RMSPE ratio here than in SCtools, so `placebo_p_value` can differ by whole ranks. -- **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — raise `n_starts` on `in_space_placebo()` or re-fit with a larger `optimizer_options['maxiter']`). +- **Note (placebo failure handling):** a placebo is **excluded from both the numerator and the denominator** of the rank (never penalized into it) and tallied in `n_failed` when its fit is not a valid optimum — EITHER its **inner Frank-Wolfe weight solve** did not converge (a truncated `W` is unusable) OR its **outer `V` search** did not converge (an under-optimized `V` fits the pre-period worse, shrinking the RMSPE ratio and biasing the p-value anti-conservatively, so it must not silently enter the rank). The reported p-value uses the **effective** count `rank / (n_placebos + 1)`, where `n_placebos` is the number of placebos that entered the reference set. Failed donors still appear in `get_placebo_df()` (`status="failed"`, NaN metrics), so once a reference set is produced the table is the full treated + every-donor unit set (`n_donors + 1` rows). In the fail-closed cases the placebo loop does not run and only the treated row is returned: `J < 2` → `placebo_p_value` is NaN with a warning (no placebo distribution; `J == 2` warns the distribution is coarse), and a treated fit whose own **inner OR outer** search did not converge also fails closed (ranking a truncated / under-optimized treated statistic would not be a valid permutation). **Caveat:** each placebo refit inherits the original fit's `optimizer_options` / `n_starts`, so valid inference requires settings adequate for the outer `V` search to converge to a comparable-quality synthetic (production defaults do; cheap settings under-optimize placebo `V` and those placebos are dropped as failed — re-fit with a larger `n_starts` or `optimizer_options['maxiter']`). **Note (same procedure for all `J+1` statistics):** under `v_method="nested"` / `"cv"`, `in_space_placebo(n_starts=k)` with `k` different from the fit's `n_starts` is rejected with a `ValueError`. More starts can select a different `V` optimum even when every fit converges, and the treated unit's ratio was produced at fit time with the fit's `n_starts`, so a different placebo budget would rank one statistic against references computed by another rule and void the exactness argument of the donor-pool Note. The remedy is one re-fit with the larger `n_starts`. `custom` / `inverse_variance` run no `V` search, so the value is inert there. - **Note (in-space / leave-one-out infeasible vs failed split):** an excluded `in_space_placebo()` / `leave_one_out()` refit is attributed to one of two causes, mirroring the split `in_time_placebo` already reports. A **solver non-convergence** (a truncated inner `W` or outer `V` search) is tallied in `n_failed` (`_loo_n_failed`) with a `status="failed"` row; a **structural cv infeasibility** — under `v_method="cv"`, the pseudo-treated donor pool (in-space) or reduced pool (leave-one-out) is indistinguishable in a re-aggregated CV window, so the weights are unidentified (the window-information gate above; `_outer_solve_V_cv` returns a structural sentinel that `_placebo_fit_unit` threads out) — is tallied in `n_infeasible` (`_loo_n_infeasible`) with a `status="infeasible"` row. Both are excluded from the rank / ATT range identically, so `placebo_p_value` / `n_placebos` are **unchanged** by the attribution — only the diagnostic accounting is refined. `_placebo_status` / `_loo_status` gain `all_placebos_infeasible` / `all_placebos_unusable` (resp. `all_refits_infeasible` / `all_refits_unusable`) codes for a no-usable-refit run that is purely structural or a mix, and `DiagnosticReport` surfaces the machine-readable `reason_code` alongside `n_failed` / `n_infeasible`. `n_infeasible` is 0 for the non-`cv` `v_method`s (no structural-identification gate). The remedy differs by cause: a structural infeasibility needs different predictors / `v_cv_t0` / donor pool, NOT a larger `n_starts` / `inner_max_iter`. - **Note (RMSPE-ratio floor):** the reported `rmspe_ratio = sqrt(MSPE_post / MSPE_pre)` floors the pre-period MSPE denominator at a scale-aware `1e-8 · max(|pre-outcomes|, 1)²` (before the square root) so a (near-)perfect pre-fit (`pre-MSPE → 0`) yields a large-but-FINITE ratio rather than `inf`/`nan` (which would corrupt the rank). Ties (`ratio_j ≥ treated_ratio`) are counted, making the p-value conservative. Mirrors the `_fit_tol` poor-fit guard. - **Note (placebo p-value is non-analytical):** `placebo_p_value` is deliberately a SEPARATE field from `p_value` (which stays NaN) — it is a permutation p-value with no SE / t-stat, so it does not flow through `safe_inference`. `is_significant` likewise stays bound to the (NaN) `p_value`, NOT `placebo_p_value`; a tool gating on `is_significant` will see `False` even when `placebo_p_value` is small. The reporting stack surfaces the placebo p-value through `estimator_native_diagnostics`, never the analytical headline. diff --git a/tests/test_methodology_synthetic_control.py b/tests/test_methodology_synthetic_control.py index 148b7ad6e..8cf6f495a 100644 --- a/tests/test_methodology_synthetic_control.py +++ b/tests/test_methodology_synthetic_control.py @@ -1511,6 +1511,38 @@ def test_in_space_placebo_rejects_invalid_n_starts(): assert res._placebo_status is None and res._placebo_df is None +def test_in_space_placebo_rejects_n_starts_mismatch_under_v_search(): + # Exactness needs every one of the J+1 statistics to come from the same procedure. + # The treated unit's ratio was produced at fit time with the fit's n_starts, and under + # a nested / cv V search more starts can land on a different V optimum, so a placebo + # budget that differs from the fit's is rejected (the remedy is one re-fit). + res = _fit_for_placebo(n_donors=4) # nested V, n_starts=1 via _FAST + with pytest.raises(ValueError, match="differs from the fit's n_starts=1"): + res.in_space_placebo(n_starts=2) + assert res._placebo_status is None and res._placebo_df is None + # The lazy reference build behind test_sharp_null / confidence_set goes through the + # same gate. + with pytest.raises(ValueError, match="differs from the fit's n_starts=1"): + res.test_sharp_null(0.0, n_starts=2) + # The fit's own value is accepted, as is the default. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.in_space_placebo(n_starts=1) + assert res._placebo_status == "ran" + + +def test_in_space_placebo_n_starts_override_inert_without_v_search(): + # custom V runs no outer search, so n_starts is inert and a differing override must + # neither raise nor change the result. + res_a, _, _ = _fit_with_placebos(run_placebo=True) + res_b, _, _ = _fit_with_placebos(run_placebo=False) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res_b.in_space_placebo(n_starts=7) + assert res_b.placebo_p_value == res_a.placebo_p_value + pd.testing.assert_frame_equal(res_b.get_placebo_df(), res_a.get_placebo_df()) + + def test_rmspe_ratio_is_root_scale(): # The reported statistic is the ROOT-scale ratio RMSPE_post/RMSPE_pre = # sqrt(MSPE_post/MSPE_pre), NOT the MSPE ratio. Hand-worked: pre-MSPE = 4, @@ -3844,7 +3876,7 @@ def test_in_space_placebo_rerun_invalidates_confidence_set(): assert native["confidence_set"]["status"] == "ran" with warnings.catch_warnings(): warnings.simplefilter("ignore") - res.in_space_placebo(n_starts=2) # rebuild the reference set + res.in_space_placebo() # explicit rebuild of the reference set assert res.effect_confidence_set is None with pytest.raises(ValueError, match="No confidence set"): res.get_confidence_set_df() From 8b88e0c6cf36e0f535c0f074d1497594330060a7 Mon Sep 17 00:00:00 2001 From: igerber Date: Wed, 14 Oct 2026 19:45:27 -0400 Subject: [PATCH 5/5] fix(synthetic_control): keep a cached confidence set when an in_space_placebo n_starts override is rejected The confidence-set cache was cleared before the n_starts validation ran, so a caller whose override was rejected (mismatch under a nested / cv V search, or a non-positive value) lost a confidence set that was still valid: nothing had been rebuilt, so nothing was stale. Clear the cache only after every argument check has passed and the reference set is certain to be rebuilt. Regression test computes a set, attempts both rejected overrides, and checks the set and its frame are untouched (and that an accepted call still clears). Docs: state the random-assignment condition on the exactness claim in llms-full.txt and the changelog fragment, as the docstring and REGISTRY already do; the changelog no longer says get_weights_df() rows follow panel order (it stays weight-sorted; donor_weights keys follow panel order) and qualifies the two-donor V search to nested / cv fits. --- .../20261005-in-space-placebo-donor-pool.md | 17 ++++++------- diff_diff/guides/llms-full.txt | 2 +- diff_diff/synthetic_control_results.py | 17 +++++++------ tests/test_methodology_synthetic_control.py | 24 +++++++++++++++++++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/changelog.d/20261005-in-space-placebo-donor-pool.md b/changelog.d/20261005-in-space-placebo-donor-pool.md index eacc2f0ee..ae196ee14 100644 --- a/changelog.d/20261005-in-space-placebo-donor-pool.md +++ b/changelog.d/20261005-in-space-placebo-donor-pool.md @@ -5,9 +5,10 @@ p. 501) do ("shifting California to the donor pool"). It used to drop the treated unit from every placebo's donor pool (the `SCtools::generate.placebos` construction); that reference set changes with the treated unit, so the - permutation p-value was not exact. (The new p-value is exact when every - placebo refit enters the reference set; failed or infeasible refits are still - excluded and reported via `n_failed` / `n_infeasible`.) Every placebo that puts weight on the + permutation p-value was not exact. (The new p-value is exact under the null + of no effect and random assignment, when every placebo refit enters the + reference set; failed or infeasible refits are still excluded and reported via + `n_failed` / `n_infeasible`.) Every placebo that puts weight on the treated unit now gets a different RMSPE ratio, so `placebo_p_value` and the placebo rows of `get_placebo_df()` change on every such fit, by whole ranks; the fit itself (`att`, weights, the treated unit's `rmspe_ratio`) does not @@ -21,13 +22,13 @@ use the treated unit's null-imputed outcome `Y_1 - f` inside each placebo's synthetic control (Firpo & Possebom 2018, fn 10), so their p-values and sets change too. With exactly 2 donors each placebo is now fit against 2 units - (a real weight solve and `V` search instead of a forced `w = [1]`); fewer - than 2 donors still gives a NaN p-value. + (a real weight solve, and under `v_method="nested"` / `"cv"` a `V` search, + instead of a forced `w = [1]`); fewer than 2 donors still gives a NaN p-value. `fit()` now processes an explicit `donor_pool` in the panel's sorted unit order (the order the placebo refits use) rather than the caller's list order, - so `donor_weights` / `get_weights_df()` rows follow that order and an exact - weight tie between donors resolves the same way in the treated fit and in - every placebo. + so `donor_weights` keys follow that order (`get_weights_df()` stays sorted by + weight descending) and an exact weight tie between donors resolves the same + way in the treated fit and in every placebo. ### Behavioral Changes - **`SyntheticControlResults.in_space_placebo(n_starts=...)` rejects a multistart diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 292693505..edabb7620 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -675,7 +675,7 @@ scm.fit( ) -> SyntheticControlResults ``` -**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits it against all the other units (the real treated unit is in every placebo pool, ADH 2010 §3.4, so the permutation is exact under the null whenever every placebo refit enters the reference set; failed / infeasible refits are excluded and shrink the effective denominator, see `n_failed` / `n_infeasible`), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). +**Inference:** NONE analytical — `se`/`t_stat`/`p_value`/`conf_int` are always NaN. `att` is the mean post-period gap. Significance via in-space placebo permutation inference: `results.in_space_placebo()` reassigns treatment to each donor, refits it against all the other units (the real treated unit is in every placebo pool, ADH 2010 §3.4, so the permutation is exact under the null of no effect and random assignment whenever every placebo refit enters the reference set; failed / infeasible refits are excluded and shrink the effective denominator, see `n_failed` / `n_infeasible`), and sets `placebo_p_value = rank/(n_placebos+1)` from the post/pre RMSPE-ratio. The permutation `placebo_p_value` is a SEPARATE field from the (NaN) `p_value`; `is_significant` stays bound to `p_value`. **ADH-2015 §4 robustness (opt-in, analytical inference unchanged):** `results.leave_one_out()` drops each reportably-weighted donor (weight > 1e-6) and re-fits (per-drop ATT/`delta_att` — large `delta_att` ⇒ single-donor dependence); `results.in_time_placebo()` backdates the intervention and checks for a spurious pre-period gap (TRUNCATE windowing — predictor windows in the held-out region are dropped); `results.regression_weights()` computes the implied regression-counterfactual weights `W^reg = X0a'(X0a X0a')^{-1}X1a` (intercept-augmented, sums to 1 at full row rank) and flags donors outside `[0,1]` — the extrapolation an OLS counterfactual incurs but the simplex-constrained SC cannot (pure linear algebra, no refit; min-norm + `_regw_rank_deficient` when `k+1>J`); `results.sparse_synthetic_control(sizes=None, max_subsets=50000)` exhaustively searches `C(J,l)` size-`l` donor subsets holding `V` FIXED at the baseline (default sweep `[1,2,3]` skips over-cap sizes with a warning; an explicitly-requested oversize `l` raises), reporting how fit/ATT degrade as the synthetic is forced sparse (+ `get_sparse_synthetic_control_gaps()`). **Confidence sets by test inversion (Firpo-Possebom 2018 §4, opt-in, also non-analytical):** `results.test_sharp_null(effect, gamma=0.1)` tests `H_0: α_1t = f(t)` by re-ranking the in-space placebo gaps (no refits; `test_sharp_null(0)` == `placebo_p_value`), and `results.confidence_set(family="constant"|"linear", gamma=0.1)` inverts it into a confidence set for the effect path (constant-effect interval / linear-slope set, strict `p>gamma`), surfaced on `effect_confidence_set` / `get_confidence_set_df()` — the analytical `conf_int` still stays NaN. **Conformal inference (Chernozhukov-Wüthrich-Zhu 2021, opt-in, also non-analytical):** unlike the Firpo path (which re-ranks the cross-unit placebo gaps), the conformal layer fits its OWN time-permutation-invariant constrained-LS proxy (eqs 3–4, no V-matrix) under the null on ALL periods and permutes residuals over time for the single treated unit. `results.conformal_test(effect, q=1, scheme="moving_block")` gives a joint sharp-null p-value (eqs 1–2; `q∈{1,2,∞}`); `results.conformal_confidence_intervals(alpha=0.1)` gives pointwise per-period CIs (Algorithm 1 — each period `t` uses `Z=(pre, t)`, the other post-periods dropped); `results.conformal_average_effect(alpha=0.1)` gives an average-effect CI by collapsing into `T*`-blocks (Appendix A.1). All three accept `alternative={"two-sided","greater","less"}` (one-sided uses the SIGNED average-effect statistic per CWZ Remark 1, q fixed at 1; CIs become half-lines `[lower,+inf)`/`(-inf,upper]`) and `covariates=[...]` (pivoted-variable matching rows stacked RAW into the constrained-LS proxy per the note after eq 6; residuals/p-value stay outcome-only). `scheme="moving_block"` (default, serial-dependence-robust) or `"iid"` (finer). Surfaced on `conformal_inference` / `get_conformal_grid_df()`; `conf_int` stays NaN. Predictor periods must lie within the pre window; `post_periods` must be a contiguous suffix cross-checked against `D` (no anticipation). **Usage:** diff --git a/diff_diff/synthetic_control_results.py b/diff_diff/synthetic_control_results.py index 93c74a508..80d70b4f0 100644 --- a/diff_diff/synthetic_control_results.py +++ b/diff_diff/synthetic_control_results.py @@ -949,13 +949,6 @@ def in_space_placebo( from diff_diff.synthetic_control import _floored_pre_mspe, _mspe, _placebo_fit_unit snap = self._fit_snapshot - # A rebuilt placebo reference set invalidates any previously computed confidence set - # (test_sharp_null / confidence_set re-rank against THIS reference set), so drop the - # cached confidence-set outputs up front — a stale set must never be reported after an - # explicit in_space_placebo() re-run (e.g. with a different n_starts). The snapshot - # check above has already passed, so the reference IS about to be rebuilt on every exit. - self.effect_confidence_set = None - self._confidence_set_df = None donors = list(snap.donor_ids) n_donors = len(donors) if n_starts is None: @@ -985,6 +978,16 @@ def in_space_placebo( "in_space_placebo() without an override." ) + # Every argument check has passed, so the reference set IS about to be rebuilt on + # every exit below. A rebuilt reference set invalidates any previously computed + # confidence set (test_sharp_null / confidence_set re-rank against THIS reference + # set), so drop the cached confidence-set outputs here - a stale set must never be + # reported after an explicit in_space_placebo() re-run. The clear sits AFTER the + # validation so a rejected override (ValueError above) leaves a valid cached set + # untouched: nothing was rebuilt, so nothing is stale. + self.effect_confidence_set = None + self._confidence_set_df = None + treated_pre = _mspe(self.gap_path, snap.pre_periods) treated_post = _mspe(self.gap_path, snap.post_periods) treated_ratio = self.rmspe_ratio diff --git a/tests/test_methodology_synthetic_control.py b/tests/test_methodology_synthetic_control.py index 8cf6f495a..c34e92e0b 100644 --- a/tests/test_methodology_synthetic_control.py +++ b/tests/test_methodology_synthetic_control.py @@ -1531,6 +1531,30 @@ def test_in_space_placebo_rejects_n_starts_mismatch_under_v_search(): assert res._placebo_status == "ran" +def test_rejected_n_starts_override_keeps_cached_confidence_set(): + # A rejected override rebuilds nothing, so a confidence set computed against the + # current reference set is still valid and must survive the ValueError (the cache + # is cleared only once every argument check has passed and the rebuild is certain). + res = _fit_for_placebo(n_donors=4) # nested V, n_starts=1 via _FAST + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.confidence_set(family="constant", gamma=0.25) + before = res.effect_confidence_set + before_df = res.get_confidence_set_df() + assert before is not None + with pytest.raises(ValueError, match="differs from the fit's n_starts=1"): + res.in_space_placebo(n_starts=2) + with pytest.raises(ValueError, match="n_starts override must be a positive integer"): + res.in_space_placebo(n_starts=0) + assert res.effect_confidence_set is before + pd.testing.assert_frame_equal(res.get_confidence_set_df(), before_df) + # An accepted call still invalidates it (the reference set IS rebuilt). + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + res.in_space_placebo() + assert res.effect_confidence_set is None + + def test_in_space_placebo_n_starts_override_inert_without_v_search(): # custom V runs no outer search, so n_starts is inert and a differing override must # neither raise nor change the result.