diff --git a/doc/_quarto-dev.yml b/doc/_quarto-dev.yml index b73319b..8a5d749 100644 --- a/doc/_quarto-dev.yml +++ b/doc/_quarto-dev.yml @@ -20,8 +20,9 @@ website: - plm/plr.qmd - plm/plr_gate.qmd - plm/plr_cate.qmd - - plm/pliv.qmd - plm/lplr.qmd + - plm/plpr.qmd + - plm/pliv.qmd # DID - did/did_pa.qmd - did/did_cs.qmd diff --git a/doc/_website.yml b/doc/_website.yml index b497b60..a6373d5 100644 --- a/doc/_website.yml +++ b/doc/_website.yml @@ -25,6 +25,7 @@ website: - plm/plr_gate.qmd - plm/plr_cate.qmd - plm/lplr.qmd + - plm/plpr.qmd - plm/pliv.qmd - text: "DID" menu: diff --git a/doc/plm/plpr.qmd b/doc/plm/plpr.qmd new file mode 100644 index 0000000..f2159dd --- /dev/null +++ b/doc/plm/plpr.qmd @@ -0,0 +1,184 @@ +--- +title: "PLPR Models" + +jupyter: python3 +--- + + +```{python} +#| echo: false + +import numpy as np +import pandas as pd +from itables import init_notebook_mode +import os +import sys + +doc_dir = os.path.abspath(os.path.join(os.getcwd(), "..")) +if doc_dir not in sys.path: + sys.path.append(doc_dir) + +from utils.style_tables import generate_and_show_styled_table + +init_notebook_mode(all_interactive=True) +``` + +## Coverage + +The simulations are based on the the [make_plpr_CP2025](https://docs.doubleml.org/stable/api/datasets.html#dataset-generators)-DGP with $1000$ units and $10$ time periods. The following DGPs are considered: + + - DGP 1: Linear in the nuisance parameters + - DGP 2: Non-linear and smooth in the nuisance parameters + - DGP 3: Non-linear and discontinuous in the nuisance parameters + + +::: {.callout-note title="Metadata" collapse="true"} + +```{python} +#| echo: false +metadata_file = '../../results/plm/plpr_ate_metadata.csv' +metadata_df = pd.read_csv(metadata_file) +print(metadata_df.T.to_string(header=False)) +``` + +::: + +```{python} +#| echo: false + +# set up data and rename columns +df_coverage = pd.read_csv("../../results/plm/plpr_ate_coverage.csv", index_col=None) + +if "repetition" in df_coverage.columns and df_coverage["repetition"].nunique() == 1: + n_rep_coverage = df_coverage["repetition"].unique()[0] +elif "n_rep" in df_coverage.columns and df_coverage["n_rep"].nunique() == 1: + n_rep_coverage = df_coverage["n_rep"].unique()[0] +else: + n_rep_coverage = "N/A" # Fallback if n_rep cannot be determined + +display_columns_coverage = ["Learner g", "Learner m", "DGP", "Approach", "Bias", "CI Length", "Coverage", "Loss g", "Loss m"] +``` + +### Partialling out + +```{python} +# | echo: false + +generate_and_show_styled_table( + main_df=df_coverage, + filters={"level": 0.95, "Score": "partialling out"}, + display_cols=display_columns_coverage, + n_rep=n_rep_coverage, + level_col="level", + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, + coverage_highlight_cols=["Coverage"] +) +``` + +```{python} +#| echo: false + +generate_and_show_styled_table( + main_df=df_coverage, + filters={"level": 0.9, "Score": "partialling out"}, + display_cols=display_columns_coverage, + n_rep=n_rep_coverage, + level_col="level", + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, + coverage_highlight_cols=["Coverage"] +) +``` + +### IV-type + +For the IV-type score, the learners `ml_l` and `ml_g` are both set to the same type of learner (here **Learner g**). + +```{python} +#| echo: false + +generate_and_show_styled_table( + main_df=df_coverage, + filters={"level": 0.95, "Score": "IV-type"}, + display_cols=display_columns_coverage, + n_rep=n_rep_coverage, + level_col="level", + coverage_highlight_cols=["Coverage"] +) +``` + +```{python} +#| echo: false + +generate_and_show_styled_table( + main_df=df_coverage, + filters={"level": 0.9, "Score": "IV-type"}, + display_cols=display_columns_coverage, + n_rep=n_rep_coverage, + level_col="level", + coverage_highlight_cols=["Coverage"] +) +``` + + +## Tuning + +The simulations are based on the the [make_plpr_CP2025](https://docs.doubleml.org/stable/api/datasets.html#dataset-generators)-DGP with $1000$ units and $10$ time periods. The following DGPs are considered: + + - DGP 1: Linear in the nuisance parameters + - DGP 3: Non-linear and discontinuous in the nuisance parameters + +This is only an example as the untuned version just relies on the default configuration. + +::: {.callout-note title="Metadata" collapse="true"} + +```{python} +#| echo: false +metadata_file = '../../results/plm/plpr_ate_tune_metadata.csv' +metadata_df = pd.read_csv(metadata_file) +print(metadata_df.T.to_string(header=False)) +``` + +::: + +```{python} +#| echo: false + +# set up data +df_tune_cov = pd.read_csv("../../results/plm/plpr_ate_tune_coverage.csv", index_col=None) + +assert df_tune_cov["repetition"].nunique() == 1 +n_rep_tune_cov = df_tune_cov["repetition"].unique()[0] + +display_columns_tune_cov = ["Learner g", "Learner m", "Tuned", "DGP", "Approach", "Bias", "CI Length", "Coverage", "Loss g", "Loss m"] +``` + + +### Partialling out + +```{python} +# | echo: false + +generate_and_show_styled_table( + main_df=df_tune_cov, + filters={"level": 0.95, "Score": "partialling out"}, + display_cols=display_columns_tune_cov, + n_rep=n_rep_tune_cov, + level_col="level", + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, + coverage_highlight_cols=["Coverage"] +) +``` + +```{python} +#| echo: false + +generate_and_show_styled_table( + main_df=df_tune_cov, + filters={"level": 0.9, "Score": "partialling out"}, + display_cols=display_columns_tune_cov, + n_rep=n_rep_tune_cov, + level_col="level", + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, + coverage_highlight_cols=["Coverage"] +) +``` diff --git a/doc/plm/plr.qmd b/doc/plm/plr.qmd index 2fe0aaf..03ccd28 100644 --- a/doc/plm/plr.qmd +++ b/doc/plm/plr.qmd @@ -50,7 +50,7 @@ elif "n_rep" in df_coverage.columns and df_coverage["n_rep"].nunique() == 1: else: n_rep_coverage = "N/A" # Fallback if n_rep cannot be determined -display_columns_coverage = ["Learner g", "Learner m", "Bias", "CI Length", "Coverage"] +display_columns_coverage = ["Learner g", "Learner m", "Bias", "CI Length", "Coverage", "Loss g", "Loss m"] ``` ### Partialling out @@ -64,7 +64,7 @@ generate_and_show_styled_table( display_cols=display_columns_coverage, n_rep=n_rep_coverage, level_col="level", - rename_map={"Learner g": "Learner l"}, + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, coverage_highlight_cols=["Coverage"] ) ``` @@ -78,7 +78,7 @@ generate_and_show_styled_table( display_cols=display_columns_coverage, n_rep=n_rep_coverage, level_col="level", - rename_map={"Learner g": "Learner l"}, + rename_map={"Learner g": "Learner l", "Loss g": "Loss l"}, coverage_highlight_cols=["Coverage"] ) ``` diff --git a/monte-cover/src/montecover/plm/__init__.py b/monte-cover/src/montecover/plm/__init__.py index 0edaedc..4be0ccb 100644 --- a/monte-cover/src/montecover/plm/__init__.py +++ b/monte-cover/src/montecover/plm/__init__.py @@ -3,6 +3,8 @@ from montecover.plm.lplr_ate import LPLRATECoverageSimulation from montecover.plm.lplr_ate_tune import LPLRATETuningCoverageSimulation from montecover.plm.pliv_late import PLIVLATECoverageSimulation +from montecover.plm.plpr_ate import PLPRATECoverageSimulation +from montecover.plm.plpr_ate_tune import PLPRATETuningCoverageSimulation from montecover.plm.plr_ate import PLRATECoverageSimulation from montecover.plm.plr_ate_sensitivity import PLRATESensitivityCoverageSimulation from montecover.plm.plr_ate_tune import PLRATETuningCoverageSimulation @@ -16,6 +18,9 @@ "PLRCATECoverageSimulation", "PLRATESensitivityCoverageSimulation", "PLRATETuningCoverageSimulation", + "PLPRATECoverageSimulation", + "PLPRATETuningCoverageSimulation", "LPLRATECoverageSimulation", "LPLRATETuningCoverageSimulation", + "PLPRATECoverageSimulation", ] diff --git a/monte-cover/src/montecover/plm/plpr_ate.py b/monte-cover/src/montecover/plm/plpr_ate.py new file mode 100644 index 0000000..cf3516f --- /dev/null +++ b/monte-cover/src/montecover/plm/plpr_ate.py @@ -0,0 +1,142 @@ +from typing import Any, Dict, Optional + +import doubleml as dml +from doubleml.plm.datasets import make_plpr_CP2025 + +from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config + + +class PLPRATECoverageSimulation(BaseSimulation): + """Simulation class for coverage properties of DoubleMLPLPR for ATE estimation.""" + + def __init__( + self, + config_file: str, + suppress_warnings: bool = True, + log_level: str = "INFO", + log_file: Optional[str] = None, + ): + super().__init__( + config_file=config_file, + suppress_warnings=suppress_warnings, + log_level=log_level, + log_file=log_file, + ) + + # Calculate oracle values + self._calculate_oracle_values() + + def _process_config_parameters(self): + """Process simulation-specific parameters from config""" + # Process ML models in parameter grid + assert ( + "learners" in self.dml_parameters + ), "No learners specified in the config file" + + required_learners = ["ml_g", "ml_m"] + for learner in self.dml_parameters["learners"]: + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" + + def _calculate_oracle_values(self): + """Calculate oracle values for the simulation.""" + self.logger.info("Calculating oracle values") + + self.oracle_values = dict() + self.oracle_values["theta"] = self.dgp_parameters["theta"] + + def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: + """Run a single repetition with the given parameters.""" + # Extract parameters + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) + score = dml_params["score"] + approach = dml_params["approach"] + + # Model + dml_model = dml.DoubleMLPLPR( + obj_dml_data=dml_data, + ml_l=ml_g, + ml_m=ml_m, + ml_g=ml_g if score == "IV-type" else None, + score=score, + approach=approach, + ) + dml_model.fit() + nuisance_loss = dml_model.nuisance_loss + + result = { + "coverage": [], + } + for level in self.confidence_parameters["level"]: + level_result = dict() + level_result["coverage"] = self._compute_coverage( + thetas=dml_model.coef, + oracle_thetas=self.oracle_values["theta"], + confint=dml_model.confint(level=level), + joint_confint=None, + ) + + # add parameters to the result + for res in level_result.values(): + res.update( + { + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Score": score, + "Approach": approach, + "level": level, + "Loss g": nuisance_loss["ml_l"].mean() if score == "partialling out" else nuisance_loss["ml_g"].mean(), + "Loss m": nuisance_loss["ml_m"].mean(), + } + ) + for key, res in level_result.items(): + result[key].append(res) + + return result + + def summarize_results(self): + """Summarize the simulation results.""" + self.logger.info("Summarizing simulation results") + + # Group by parameter combinations + groupby_cols = ["Learner g", "Learner m", "Score", "Approach", "DGP", "level"] + aggregation_dict = { + "Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "Loss g": "mean", + "Loss m": "mean", + "repetition": "count", + } + + # Aggregate results (possibly multiple result dfs) + result_summary = dict() + for result_name, result_df in self.results.items(): + result_summary[result_name] = ( + result_df.groupby(groupby_cols).agg(aggregation_dict).reset_index() + ) + self.logger.debug(f"Summarized {result_name} results") + + return result_summary + + def _generate_dml_data(self, dgp_params) -> dml.DoubleMLData: + """Generate data for the simulation.""" + data = make_plpr_CP2025( + num_id=dgp_params["num_id"], + num_t=dgp_params["num_t"], + dim_x=dgp_params["dim_x"], + theta=dgp_params["theta"], + dgp_type=dgp_params["DGP"], + ) + dml_data = dml.DoubleMLPanelData( + data, + y_col="y", + d_cols="d", + t_col="time", + id_col="id", + static_panel=True, + ) + return dml_data diff --git a/monte-cover/src/montecover/plm/plpr_ate_tune.py b/monte-cover/src/montecover/plm/plpr_ate_tune.py new file mode 100644 index 0000000..dde0230 --- /dev/null +++ b/monte-cover/src/montecover/plm/plpr_ate_tune.py @@ -0,0 +1,168 @@ +from typing import Any, Dict, Optional + +import doubleml as dml +import optuna +from doubleml.plm.datasets import make_plpr_CP2025 + +from montecover.base import BaseSimulation +from montecover.utils import create_learner_from_config +from montecover.utils_tuning import lgbm_reg_params + + +class PLPRATETuningCoverageSimulation(BaseSimulation): + """Simulation class for coverage properties of DoubleMLPLPR for ATE estimation with tuning.""" + + def __init__( + self, + config_file: str, + suppress_warnings: bool = True, + log_level: str = "INFO", + log_file: Optional[str] = None, + ): + super().__init__( + config_file=config_file, + suppress_warnings=suppress_warnings, + log_level=log_level, + log_file=log_file, + ) + + # Calculate oracle values + self._calculate_oracle_values() + # tuning specific settings + self._param_space = {"ml_l": lgbm_reg_params, "ml_m": lgbm_reg_params} + + self._optuna_settings = { + "n_trials": 50, + "show_progress_bar": False, + "verbosity": optuna.logging.WARNING, # Suppress Optuna logs + } + + def _process_config_parameters(self): + """Process simulation-specific parameters from config""" + # Process ML models in parameter grid + assert ( + "learners" in self.dml_parameters + ), "No learners specified in the config file" + + required_learners = ["ml_g", "ml_m"] + for learner in self.dml_parameters["learners"]: + for ml in required_learners: + assert ml in learner, f"No {ml} specified in the config file" + + def _calculate_oracle_values(self): + """Calculate oracle values for the simulation.""" + self.logger.info("Calculating oracle values") + + self.oracle_values = dict() + self.oracle_values["theta"] = self.dgp_parameters["theta"] + + def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: + """Run a single repetition with the given parameters.""" + # Extract parameters + learner_config = dml_params["learners"] + learner_g_name, ml_g = create_learner_from_config(learner_config["ml_g"]) + learner_m_name, ml_m = create_learner_from_config(learner_config["ml_m"]) + score = dml_params["score"] + approach = dml_params["approach"] + + # Model + dml_model = dml.DoubleMLPLPR( + obj_dml_data=dml_data, + ml_l=ml_g, + ml_m=ml_m, + ml_g=ml_g if score == "IV-type" else None, + score=score, + approach=approach, + ) + + dml_model_tuned = dml.DoubleMLPLPR( + obj_dml_data=dml_data, + ml_l=ml_g, + ml_m=ml_m, + ml_g=ml_g if score == "IV-type" else None, + score=score, + approach=approach, + ) + dml_model_tuned.tune_ml_models( + ml_param_space=self._param_space, + optuna_settings=self._optuna_settings, + ) + + result = { + "coverage": [], + } + for model in [dml_model, dml_model_tuned]: + model.fit() + nuisance_loss = model.nuisance_loss + + for level in self.confidence_parameters["level"]: + level_result = dict() + level_result["coverage"] = self._compute_coverage( + thetas=model.coef, + oracle_thetas=self.oracle_values["theta"], + confint=model.confint(level=level), + joint_confint=None, + ) + + # add parameters to the result + for res in level_result.values(): + res.update( + { + "Learner g": learner_g_name, + "Learner m": learner_m_name, + "Score": score, + "Approach": approach, + "level": level, + "Tuned": model is dml_model_tuned, + "Loss g": nuisance_loss["ml_l"].mean() if score == "partialling out" else nuisance_loss["ml_g"].mean(), + "Loss m": nuisance_loss["ml_m"].mean(), + } + ) + for key, res in level_result.items(): + result[key].append(res) + + return result + + def summarize_results(self): + """Summarize the simulation results.""" + self.logger.info("Summarizing simulation results") + + # Group by parameter combinations + groupby_cols = ["Learner g", "Learner m", "Score", "Approach", "DGP", "level", "Tuned"] + aggregation_dict = { + "Coverage": "mean", + "CI Length": "mean", + "Bias": "mean", + "Loss g": "mean", + "Loss m": "mean", + "repetition": "count", + } + + # Aggregate results (possibly multiple result dfs) + result_summary = dict() + for result_name, result_df in self.results.items(): + result_summary[result_name] = ( + result_df.groupby(groupby_cols).agg(aggregation_dict).reset_index() + ) + self.logger.debug(f"Summarized {result_name} results") + + return result_summary + + def _generate_dml_data(self, dgp_params) -> dml.DoubleMLData: + """Generate data for the simulation.""" + data = make_plpr_CP2025( + num_id=dgp_params["num_id"], + num_t=dgp_params["num_t"], + dim_x=dgp_params["dim_x"], + theta=dgp_params["theta"], + dgp_type=dgp_params["DGP"], + ) + dml_data = dml.DoubleMLPanelData( + data, + y_col="y", + d_cols="d", + t_col="time", + id_col="id", + static_panel=True, + ) + return dml_data diff --git a/monte-cover/src/montecover/plm/plr_ate.py b/monte-cover/src/montecover/plm/plr_ate.py index 0f19214..dc56746 100644 --- a/monte-cover/src/montecover/plm/plr_ate.py +++ b/monte-cover/src/montecover/plm/plr_ate.py @@ -63,6 +63,7 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: score=score, ) dml_model.fit() + nuisance_loss = dml_model.nuisance_loss result = { "coverage": [], @@ -84,6 +85,8 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: "Learner m": learner_m_name, "Score": score, "level": level, + "Loss g": nuisance_loss["ml_l"].mean() if score == "partialling out" else nuisance_loss["ml_g"].mean(), + "Loss m": nuisance_loss["ml_m"].mean(), } ) for key, res in level_result.items(): @@ -101,6 +104,8 @@ def summarize_results(self): "Coverage": "mean", "CI Length": "mean", "Bias": "mean", + "Loss g": "mean", + "Loss m": "mean", "repetition": "count", } diff --git a/monte-cover/src/montecover/plm/plr_ate_tune.py b/monte-cover/src/montecover/plm/plr_ate_tune.py index c50b5ef..6fbe843 100644 --- a/monte-cover/src/montecover/plm/plr_ate_tune.py +++ b/monte-cover/src/montecover/plm/plr_ate_tune.py @@ -73,7 +73,6 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: ml_g=ml_g if score == "IV-type" else None, score=score, ) - dml_model.fit() dml_model_tuned = dml.DoubleMLPLR( obj_dml_data=dml_data, @@ -86,12 +85,12 @@ def run_single_rep(self, dml_data, dml_params) -> Dict[str, Any]: ml_param_space=self._param_space, optuna_settings=self._optuna_settings, ) - dml_model_tuned.fit() result = { "coverage": [], } for model in [dml_model, dml_model_tuned]: + model.fit() nuisance_loss = model.nuisance_loss for level in self.confidence_parameters["level"]: level_result = dict() diff --git a/results/plm/plpr_ate_config.yml b/results/plm/plpr_ate_config.yml new file mode 100644 index 0000000..4ecda65 --- /dev/null +++ b/results/plm/plpr_ate_config.yml @@ -0,0 +1,41 @@ +simulation_parameters: + repetitions: 500 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + dim_x: + - 10 + num_id: + - 1000 + num_t: + - 10 + DGP: + - dgp1 + - dgp2 + - dgp3 +learner_definitions: + lasso: &id001 + name: LassoCV + lgbm: &id002 + name: LGBM Regr. +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + - ml_g: *id002 + ml_m: *id002 + score: + - partialling out + - IV-type + approach: + - cre_general + - cre_normal + - fd_exact + - wg_approx +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plpr_ate_coverage.csv b/results/plm/plpr_ate_coverage.csv new file mode 100644 index 0000000..93eade8 --- /dev/null +++ b/results/plm/plpr_ate_coverage.csv @@ -0,0 +1,97 @@ +Learner g,Learner m,Score,Approach,DGP,level,Coverage,CI Length,Bias,Loss g,Loss m,repetition +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp1,0.9,0.76,0.03514583618402707,0.012086933097448243,1.4920664942490383,0.9826812820928352,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp1,0.95,0.822,0.04187884684603002,0.012086933097448243,1.4920664942490383,0.9826812820928352,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp2,0.9,0.902,0.0352112355264963,0.00848052574435656,1.4868097497063664,0.9728827523238984,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp2,0.95,0.942,0.0419567749690873,0.00848052574435656,1.4868097497063664,0.9728827523238984,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp3,0.9,0.0,0.04310727857260173,0.14671677416272066,1.571773745295489,1.0622433672042766,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_general,dgp3,0.95,0.0,0.05136549057585367,0.14671677416272066,1.571773745295489,1.0622433672042766,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp1,0.9,0.232,0.038406550228071955,0.02742550727028207,1.483764048404902,0.999212223907986,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp1,0.95,0.364,0.04576422727471694,0.02742550727028207,1.483764048404902,0.999212223907986,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp2,0.9,0.86,0.04234693897695462,0.011253246849524612,1.4856533799380605,0.9964954831115506,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp2,0.95,0.924,0.05045949006670804,0.011253246849524612,1.4856533799380605,0.9964954831115506,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp3,0.9,0.0,0.049156980333193705,0.10411713452491797,1.5940307485754293,1.2923084248152257,500 +LGBM Regr.,LGBM Regr.,IV-type,cre_normal,dgp3,0.95,0.0,0.05857415484462794,0.10411713452491797,1.5940307485754293,1.2923084248152257,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp1,0.9,0.62,0.04102050808679343,0.017937560761299848,1.4771745005369754,1.4770380880496619,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp1,0.95,0.732,0.0488789501754378,0.017937560761299848,1.4771745005369754,1.4770380880496619,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp2,0.9,0.882,0.04190905372606799,0.010488386999141841,1.4488914582957235,1.4558939353569655,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp2,0.95,0.934,0.04993771760803054,0.010488386999141841,1.4488914582957235,1.4558939353569655,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp3,0.9,0.0,0.04370250031217863,0.1707879261368483,1.6051273815372227,1.632153918330209,500 +LGBM Regr.,LGBM Regr.,IV-type,fd_exact,dgp3,0.95,0.0,0.052074741024204015,0.1707879261368483,1.6051273815372227,1.632153918330209,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp1,0.9,0.86,0.03552645391101902,0.009773574848443971,0.975237573251918,0.9746874056099604,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp1,0.95,0.91,0.0423323808411274,0.009773574848443971,0.975237573251918,0.9746874056099604,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp2,0.9,0.9,0.03215088012049884,0.00768267021100153,0.9766529183354105,1.0740756727445167,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp2,0.95,0.94,0.03831013658293233,0.00768267021100153,0.9766529183354105,1.0740756727445167,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp3,0.9,0.0,0.03145360311737441,0.6525229675042551,1.254590697442093,1.6629745312688475,500 +LGBM Regr.,LGBM Regr.,IV-type,wg_approx,dgp3,0.95,0.0,0.03747927978754392,0.6525229675042551,1.254590697442093,1.6629745312688475,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.9,0.884,0.035521703515894315,0.009263409558316846,1.7589959457732671,0.9827241499397525,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.95,0.934,0.04232672039620745,0.009263409558316846,1.7589959457732671,0.9827241499397525,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp2,0.9,0.638,0.0348786728934553,0.01503203017611916,1.7526622862628904,0.9728084396340184,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp2,0.95,0.764,0.0415605021502273,0.01503203017611916,1.7526622862628904,0.9728084396340184,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.9,0.0,0.06598788253706468,0.13048470795365294,1.9493882632563304,1.0624147992608393,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.95,0.0,0.0786294118026844,0.13048470795365294,1.9493882632563304,1.0624147992608393,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.9,0.092,0.0450048480961515,0.03780612634140066,1.7583565503328924,0.9991400420492937,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.95,0.172,0.05362658412447015,0.03780612634140066,1.7583565503328924,0.9991400420492937,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp2,0.9,0.902,0.04599869759031349,0.011016558307233774,1.7513679581585158,0.996358190008618,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp2,0.95,0.94,0.05481082883943669,0.011016558307233774,1.7513679581585158,0.996358190008618,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.9,0.426,0.06931454796040375,0.04004134564072659,1.9470478710807493,1.2921443410214415,500 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.95,0.546,0.08259337814687731,0.04004134564072659,1.9470478710807493,1.2921443410214415,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.9,0.868,0.04160542943340649,0.010766955606101051,1.6715896763026044,1.4773030644679437,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.95,0.93,0.04957592694854741,0.010766955606101051,1.6715896763026044,1.4773030644679437,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp2,0.9,0.706,0.04117746823251659,0.015444730320919184,1.6238667435533736,1.4555522656672797,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp2,0.95,0.818,0.04906597972480611,0.015444730320919184,1.6238667435533736,1.4555522656672797,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.9,0.0,0.06904324805844071,0.15128321194444963,1.9891080939323569,1.6328417545770315,500 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.95,0.0,0.08227010437458288,0.15128321194444963,1.9891080939323569,1.6328417545770315,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.9,0.872,0.03490468627660247,0.009087617206985168,1.094306952229372,0.9748184939426158,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.95,0.942,0.041591499008092996,0.009087617206985168,1.094306952229372,0.9748184939426158,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp2,0.9,0.716,0.03126496342246468,0.011800053579619845,1.1147066793312614,1.0741359418425813,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp2,0.95,0.812,0.03725450172704079,0.011800053579619845,1.1147066793312614,1.0741359418425813,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.9,0.0,0.036138092388723404,0.6322847737379714,2.3054834813847025,1.6628430322181407,500 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.95,0.0,0.04306119303949992,0.6322847737379714,2.3054834813847025,1.6628430322181407,500 +LassoCV,LassoCV,IV-type,cre_general,dgp1,0.9,0.896,0.03276473125839177,0.00821022405330175,1.4246934279489174,0.9489026976001401,500 +LassoCV,LassoCV,IV-type,cre_general,dgp1,0.95,0.936,0.03904158532853829,0.00821022405330175,1.4246934279489174,0.9489026976001401,500 +LassoCV,LassoCV,IV-type,cre_general,dgp2,0.9,0.89,0.028861438302812854,0.007018329498324938,1.4393292657687904,1.1603587931918957,500 +LassoCV,LassoCV,IV-type,cre_general,dgp2,0.95,0.942,0.03439052490061289,0.007018329498324938,1.4393292657687904,1.1603587931918957,500 +LassoCV,LassoCV,IV-type,cre_general,dgp3,0.9,0.0,0.020298215175364506,0.8586923970192432,1.8143505199391432,2.5595991769863593,500 +LassoCV,LassoCV,IV-type,cre_general,dgp3,0.95,0.0,0.024186815192725072,0.8586923970192432,1.8143505199391432,2.5595991769863593,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp1,0.9,0.912,0.03411747806382867,0.008244289592564806,1.4231264360628921,0.9487404427482624,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp1,0.95,0.942,0.040653482566939896,0.008244289592564806,1.4231264360628921,0.9487404427482624,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp2,0.9,0.89,0.02865577716025719,0.007167442886059286,1.4378632135082987,1.1605732873412775,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp2,0.95,0.934,0.03414546453425339,0.007167442886059286,1.4378632135082987,1.1605732873412775,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp3,0.9,0.0,0.020320839419312797,0.8575641005497477,1.8154846352374259,2.5604995833015938,500 +LassoCV,LassoCV,IV-type,cre_normal,dgp3,0.95,0.0,0.024213773642151538,0.8575641005497477,1.8154846352374259,2.5604995833015938,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp1,0.9,0.894,0.0399796161499072,0.009748362312755216,1.4151388592061662,1.414574320153688,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp1,0.95,0.936,0.047638650932595,0.009748362312755216,1.4151388592061662,1.414574320153688,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp2,0.9,0.89,0.03472218740166541,0.008339407196285804,1.442296972916924,1.7298507312867564,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp2,0.95,0.944,0.04137403818590503,0.008339407196285804,1.442296972916924,1.7298507312867564,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp3,0.9,0.0,0.023598130929631075,0.8587681734465551,1.9559180380947834,3.8155575850577184,500 +LassoCV,LassoCV,IV-type,fd_exact,dgp3,0.95,0.0,0.028118907340259135,0.8587681734465551,1.9559180380947834,3.8155575850577184,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp1,0.9,0.904,0.033671023532489415,0.008213981918648031,0.9490845712902075,0.9485363031047026,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp1,0.95,0.942,0.040121499180806375,0.008213981918648031,0.9490845712902075,0.9485363031047026,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp2,0.9,0.894,0.02886848144391098,0.007003403091165492,0.9672907575243995,1.1602428664343556,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp2,0.95,0.946,0.034398917320864864,0.007003403091165492,0.9672907575243995,1.1602428664343556,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp3,0.9,0.0,0.020294054418133808,0.8587746000766776,1.3121259699415952,2.5596321343060358,500 +LassoCV,LassoCV,IV-type,wg_approx,dgp3,0.95,0.0,0.024181857344691086,0.8587746000766776,1.3121259699415952,2.5596321343060358,500 +LassoCV,LassoCV,partialling out,cre_general,dgp1,0.9,0.91,0.03467943098074278,0.008251240800288466,1.6750736379383766,0.9489001592191306,500 +LassoCV,LassoCV,partialling out,cre_general,dgp1,0.95,0.952,0.0413230907680053,0.008251240800288466,1.6750736379383766,0.9489001592191306,500 +LassoCV,LassoCV,partialling out,cre_general,dgp2,0.9,0.89,0.028881863703602084,0.0070518376131271324,1.7287268520665562,1.1603520510978118,500 +LassoCV,LassoCV,partialling out,cre_general,dgp2,0.95,0.944,0.034414863266812,0.0070518376131271324,1.7287268520665562,1.1603520510978118,500 +LassoCV,LassoCV,partialling out,cre_general,dgp3,0.9,0.0,0.020370479530043516,0.8585524814448288,4.176721826273227,2.559726882415026,500 +LassoCV,LassoCV,partialling out,cre_general,dgp3,0.95,0.0,0.02427292348237234,0.8585524814448288,4.176721826273227,2.559726882415026,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp1,0.9,0.806,0.03478280442865251,0.010499227810699951,1.6752650843495283,0.9487438547123895,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp1,0.95,0.89,0.041446267828590504,0.010499227810699951,1.6752650843495283,0.9487438547123895,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp2,0.9,0.804,0.0289631665115877,0.009127066587909956,1.7289940040116214,1.1605626902587372,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp2,0.95,0.856,0.03451174153785252,0.009127066587909956,1.7289940040116214,1.1605626902587372,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp3,0.9,0.0,0.02035309555522939,0.8624902127509477,4.176610937542443,2.560480949006136,500 +LassoCV,LassoCV,partialling out,cre_normal,dgp3,0.95,0.0,0.024252209198751252,0.8624902127509477,4.176610937542443,2.560480949006136,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp1,0.9,0.904,0.041693471947824016,0.009706029679925616,1.5821336249148614,1.4145383386834307,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp1,0.95,0.956,0.04968083607513426,0.009706029679925616,1.5821336249148614,1.4145383386834307,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp2,0.9,0.896,0.03473840962559508,0.008340460416278743,1.682105933852392,1.7298235340863823,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp2,0.95,0.944,0.04139336815796475,0.008340460416278743,1.682105933852392,1.7298235340863823,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp3,0.9,0.0,0.02374608866600501,0.8586899397478316,5.543468443960648,3.8154775099637335,500 +LassoCV,LassoCV,partialling out,fd_exact,dgp3,0.95,0.0,0.028295209857258443,0.8586899397478316,5.543468443960648,3.8154775099637335,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp1,0.9,0.91,0.03469221899156352,0.008229913285302625,1.0609031375206512,0.9485499616068327,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp1,0.95,0.95,0.04133832862274926,0.008229913285302625,1.0609031375206512,0.9485499616068327,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp2,0.9,0.892,0.028882696610382973,0.007030114956002142,1.1279298355369647,1.1602246284252973,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp2,0.95,0.946,0.03441585573645566,0.007030114956002142,1.1279298355369647,1.1602246284252973,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp3,0.9,0.0,0.02037307527005654,0.8587768373127108,3.71824421002822,2.5596448613420026,500 +LassoCV,LassoCV,partialling out,wg_approx,dgp3,0.95,0.0,0.024276016497370993,0.8587768373127108,3.71824421002822,2.5596448613420026,500 diff --git a/results/plm/plpr_ate_metadata.csv b/results/plm/plpr_ate_metadata.csv new file mode 100644 index 0000000..e8b0af6 --- /dev/null +++ b/results/plm/plpr_ate_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.2.dev96,PLPRATECoverageSimulation,2026-01-16 15:02,36.614749721686046,3.12.9,scripts/plm/plpr_ate_config.yml diff --git a/results/plm/plpr_ate_tune_config.yml b/results/plm/plpr_ate_tune_config.yml new file mode 100644 index 0000000..40ecc47 --- /dev/null +++ b/results/plm/plpr_ate_tune_config.yml @@ -0,0 +1,35 @@ +simulation_parameters: + repetitions: 100 + max_runtime: 19800 + random_seed: 42 + n_jobs: -2 +dgp_parameters: + theta: + - 0.5 + dim_x: + - 10 + num_id: + - 1000 + num_t: + - 10 + DGP: + - dgp1 + - dgp3 +learner_definitions: + lgbm: &id001 + name: LGBM Regr. +dml_parameters: + learners: + - ml_g: *id001 + ml_m: *id001 + score: + - partialling out + approach: + - cre_general + - cre_normal + - fd_exact + - wg_approx +confidence_parameters: + level: + - 0.95 + - 0.9 diff --git a/results/plm/plpr_ate_tune_coverage.csv b/results/plm/plpr_ate_tune_coverage.csv new file mode 100644 index 0000000..d2cbb9d --- /dev/null +++ b/results/plm/plpr_ate_tune_coverage.csv @@ -0,0 +1,33 @@ +Learner g,Learner m,Score,Approach,DGP,level,Tuned,Coverage,CI Length,Bias,Loss g,Loss m,repetition +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.9,False,0.85,0.03536060813686728,0.010137837243259291,1.759385900869758,0.9829393756223068,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.9,True,0.43,0.03545698050140429,0.019649649061510455,1.7952996538071346,1.0074865255650423,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.95,False,0.88,0.042134763412439845,0.010137837243259291,1.759385900869758,0.9829393756223068,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp1,0.95,True,0.6,0.04224959816764391,0.019649649061510455,1.7952996538071346,1.0074865255650423,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.9,False,0.0,0.06858490422345778,0.13496018593341227,1.9519737434421347,1.065153140737538,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.9,True,0.02,0.06394880355115651,0.06224369604076415,1.9303888300716567,1.0663837612950566,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.95,False,0.0,0.08172395400935707,0.13496018593341227,1.9519737434421347,1.065153140737538,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_general,dgp3,0.95,True,0.11,0.07619969932947211,0.06224369604076415,1.9303888300716567,1.0663837612950566,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.9,False,0.13,0.04499659790967875,0.037967155486427345,1.7597864638844198,1.000259350721019,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.9,True,0.05,0.04578777654320108,0.05361966770305959,1.7958356986277106,0.9953445685695448,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.95,False,0.19,0.053616753420942846,0.037967155486427345,1.7597864638844198,1.000259350721019,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp1,0.95,True,0.06,0.05455950091022274,0.05361966770305959,1.7958356986277106,0.9953445685695448,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.9,False,0.38,0.07033182725843065,0.0414062764743804,1.9511557372062653,1.292533000356088,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.9,True,0.75,0.07270339833567722,0.024516389764608012,1.9331219261429609,1.2700435982351912,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.95,False,0.5,0.08380554119511537,0.0414062764743804,1.9511557372062653,1.292533000356088,100 +LGBM Regr.,LGBM Regr.,partialling out,cre_normal,dgp3,0.95,True,0.85,0.08663144243156458,0.024516389764608012,1.9331219261429609,1.2700435982351912,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.9,False,0.9,0.04138210865225103,0.010006694800155022,1.6707591787769969,1.477648072202423,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.9,True,0.85,0.041322602449946635,0.011232911566422777,1.6567152207263192,1.467277534675784,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.95,False,0.94,0.04930982382490638,0.010006694800155022,1.6707591787769969,1.477648072202423,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp1,0.95,True,0.92,0.04923891781147008,0.011232911566422777,1.6567152207263192,1.467277534675784,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.9,False,0.0,0.06993795589874738,0.15347152370108225,1.9933735415428915,1.6344498070427307,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.9,True,0.0,0.06974766121306378,0.08962925447476038,1.9625374204301658,1.642912272067501,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.95,False,0.0,0.0833362145225945,0.15347152370108225,1.9933735415428915,1.6344498070427307,100 +LGBM Regr.,LGBM Regr.,partialling out,fd_exact,dgp3,0.95,True,0.0,0.08310946441895123,0.08962925447476038,1.9625374204301658,1.642912272067501,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.9,False,0.84,0.03477046525415805,0.00980255121555226,1.0936318545403032,0.9752601991470606,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.9,True,0.86,0.03476589854784296,0.009227874716762542,1.0883774496861216,0.9693908447489228,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.95,False,0.89,0.04143156479531641,0.00980255121555226,1.0936318545403032,0.9752601991470606,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp1,0.95,True,0.93,0.04142612322911326,0.009227874716762542,1.0883774496861216,0.9693908447489228,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.9,False,0.0,0.03627472064440453,0.6316795367533637,2.3058431372679435,1.6644837881054577,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.9,True,0.0,0.03648420793166297,0.6110007119820693,2.364071320754483,1.7088324600649871,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.95,False,0.0,0.043223995647596956,0.6316795367533637,2.3058431372679435,1.6644837881054577,100 +LGBM Regr.,LGBM Regr.,partialling out,wg_approx,dgp3,0.95,True,0.0,0.04347361514657118,0.6110007119820693,2.364071320754483,1.7088324600649871,100 diff --git a/results/plm/plpr_ate_tune_metadata.csv b/results/plm/plpr_ate_tune_metadata.csv new file mode 100644 index 0000000..5795390 --- /dev/null +++ b/results/plm/plpr_ate_tune_metadata.csv @@ -0,0 +1,2 @@ +DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File +0.11.2.dev96,PLPRATETuningCoverageSimulation,2026-01-16 16:54,76.14726469516754,3.12.9,scripts/plm/plpr_ate_tune_config.yml diff --git a/results/plm/plr_ate_coverage.csv b/results/plm/plr_ate_coverage.csv index 305cb65..6576238 100644 --- a/results/plm/plr_ate_coverage.csv +++ b/results/plm/plr_ate_coverage.csv @@ -1,29 +1,29 @@ -Learner g,Learner m,Score,level,Coverage,CI Length,Bias,repetition -LGBM Regr.,LGBM Regr.,IV-type,0.9,0.889,0.15978309138485988,0.040678761104429376,1000 -LGBM Regr.,LGBM Regr.,IV-type,0.95,0.939,0.1903932965957687,0.040678761104429376,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.9,0.838,0.14681770731506133,0.041275148836012944,1000 -LGBM Regr.,LGBM Regr.,partialling out,0.95,0.897,0.17494408858956342,0.041275148836012944,1000 -LGBM Regr.,LassoCV,IV-type,0.9,0.876,0.14833747639697978,0.039489073815308307,1000 -LGBM Regr.,LassoCV,IV-type,0.95,0.931,0.17675500514564524,0.039489073815308307,1000 -LGBM Regr.,LassoCV,partialling out,0.9,0.893,0.15937942718304723,0.03925857893846666,1000 -LGBM Regr.,LassoCV,partialling out,0.95,0.94,0.18991230103212867,0.03925857893846666,1000 -LassoCV,LGBM Regr.,IV-type,0.9,0.878,0.15066272181786255,0.03855046266735263,1000 -LassoCV,LGBM Regr.,IV-type,0.95,0.929,0.17952570595784667,0.03855046266735263,1000 -LassoCV,LGBM Regr.,partialling out,0.9,0.526,0.13905172633298185,0.06966076055623882,1000 -LassoCV,LGBM Regr.,partialling out,0.95,0.642,0.1656903514909566,0.06966076055623882,1000 -LassoCV,LassoCV,IV-type,0.9,0.877,0.13992145634265207,0.0366445077678926,1000 -LassoCV,LassoCV,IV-type,0.95,0.928,0.16672669871802667,0.0366445077678926,1000 -LassoCV,LassoCV,partialling out,0.9,0.897,0.14673266782178596,0.03648225251216086,1000 -LassoCV,LassoCV,partialling out,0.95,0.945,0.17484275778337358,0.03648225251216086,1000 -LassoCV,RF Regr.,IV-type,0.9,0.855,0.13032149928473868,0.037166678834479695,1000 -LassoCV,RF Regr.,IV-type,0.95,0.907,0.15528764433753836,0.037166678834479695,1000 -LassoCV,RF Regr.,partialling out,0.9,0.775,0.14270313357843983,0.04570634853982781,1000 -LassoCV,RF Regr.,partialling out,0.95,0.862,0.1700412715830077,0.04570634853982781,1000 -RF Regr.,LassoCV,IV-type,0.9,0.873,0.14101035943967402,0.03685517557846389,1000 -RF Regr.,LassoCV,IV-type,0.95,0.937,0.16802420678673716,0.03685517557846389,1000 -RF Regr.,LassoCV,partialling out,0.9,0.895,0.15051386498604236,0.037526282976942105,1000 -RF Regr.,LassoCV,partialling out,0.95,0.938,0.1793483321025444,0.037526282976942105,1000 -RF Regr.,RF Regr.,IV-type,0.9,0.837,0.13142137550510022,0.037870665143070414,1000 -RF Regr.,RF Regr.,IV-type,0.95,0.924,0.15659822768917436,0.037870665143070414,1000 -RF Regr.,RF Regr.,partialling out,0.9,0.884,0.1422661253784088,0.03699055268836001,1000 -RF Regr.,RF Regr.,partialling out,0.95,0.936,0.16952054419488322,0.03699055268836001,1000 +Learner g,Learner m,Score,level,Coverage,CI Length,Bias,Loss g,Loss m,repetition +LGBM Regr.,LGBM Regr.,IV-type,0.9,0.883,0.15928202279086665,0.041454930462256984,1.069765598118696,1.0865571968126624,1000 +LGBM Regr.,LGBM Regr.,IV-type,0.95,0.937,0.18979623654013872,0.041454930462256984,1.069765598118696,1.0865571968126624,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.9,0.842,0.14626443603990613,0.04159012092162225,1.2055345892536342,1.086959351307916,1000 +LGBM Regr.,LGBM Regr.,partialling out,0.95,0.907,0.17428482520270844,0.04159012092162225,1.2055345892536342,1.086959351307916,1000 +LGBM Regr.,LassoCV,IV-type,0.9,0.887,0.147905408377066,0.03847587695058279,1.0693318230438509,1.0086829211963027,1000 +LGBM Regr.,LassoCV,IV-type,0.95,0.948,0.17624016434520753,0.03847587695058279,1.0693318230438509,1.0086829211963027,1000 +LGBM Regr.,LassoCV,partialling out,0.9,0.899,0.15875140474791682,0.03924849266305699,1.2056443309418285,1.0084514460661456,1000 +LGBM Regr.,LassoCV,partialling out,0.95,0.951,0.189163966144349,0.03924849266305699,1.2056443309418285,1.0084514460661456,1000 +LassoCV,LGBM Regr.,IV-type,0.9,0.894,0.15032363130014348,0.03814520647938802,1.0109435769879314,1.0867218776097132,1000 +LassoCV,LGBM Regr.,IV-type,0.95,0.938,0.1791216546846278,0.03814520647938802,1.0109435769879314,1.0867218776097132,1000 +LassoCV,LGBM Regr.,partialling out,0.9,0.498,0.13861728700492693,0.07095098845194882,1.1287372870798746,1.0879589553258844,1000 +LassoCV,LGBM Regr.,partialling out,0.95,0.624,0.16517268510259017,0.07095098845194882,1.1287372870798746,1.0879589553258844,1000 +LassoCV,LassoCV,IV-type,0.9,0.876,0.1395941022128785,0.03614102732438801,1.008526334816832,1.00850722502704,1000 +LassoCV,LassoCV,IV-type,0.95,0.939,0.16633663221360717,0.03614102732438801,1.008526334816832,1.00850722502704,1000 +LassoCV,LassoCV,partialling out,0.9,0.908,0.14656359193331914,0.0355408406947952,1.1286065731474382,1.0083310548041704,1000 +LassoCV,LassoCV,partialling out,0.95,0.957,0.17464129143608328,0.0355408406947952,1.1286065731474382,1.0083310548041704,1000 +LassoCV,RF Regr.,IV-type,0.9,0.844,0.12999176189584266,0.036265043441751715,1.0092786969519125,1.0524458002714516,1000 +LassoCV,RF Regr.,IV-type,0.95,0.908,0.1548947380047176,0.036265043441751715,1.0092786969519125,1.0524458002714516,1000 +LassoCV,RF Regr.,partialling out,0.9,0.78,0.1426021188866897,0.04661398743097048,1.128740578878581,1.0523978387050963,1000 +LassoCV,RF Regr.,partialling out,0.95,0.863,0.16992090515374272,0.04661398743097048,1.128740578878581,1.0523978387050963,1000 +RF Regr.,LassoCV,IV-type,0.9,0.885,0.1406646493212081,0.03547371211869198,1.0162369561093956,1.0085999646131845,1000 +RF Regr.,LassoCV,IV-type,0.95,0.941,0.16761226777272276,0.03547371211869198,1.0162369561093956,1.0085999646131845,1000 +RF Regr.,LassoCV,partialling out,0.9,0.895,0.15031102850606376,0.03675855799694686,1.153850269585127,1.0083949164585309,1000 +RF Regr.,LassoCV,partialling out,0.95,0.946,0.17910663752924322,0.03675855799694686,1.153850269585127,1.0083949164585309,1000 +RF Regr.,RF Regr.,IV-type,0.9,0.845,0.13119958082752003,0.036530036698981375,1.0166818516109433,1.0524053946556322,1000 +RF Regr.,RF Regr.,IV-type,0.95,0.917,0.15633394303010376,0.036530036698981375,1.0166818516109433,1.0524053946556322,1000 +RF Regr.,RF Regr.,partialling out,0.9,0.888,0.14202142008522262,0.03530584490921706,1.1540093754976348,1.0526197092895246,1000 +RF Regr.,RF Regr.,partialling out,0.95,0.937,0.16922895985350928,0.03530584490921706,1.1540093754976348,1.0526197092895246,1000 diff --git a/results/plm/plr_ate_metadata.csv b/results/plm/plr_ate_metadata.csv index 7c4c7b6..f6e9b21 100644 --- a/results/plm/plr_ate_metadata.csv +++ b/results/plm/plr_ate_metadata.csv @@ -1,2 +1,2 @@ DoubleML Version,Script,Date,Total Runtime (minutes),Python Version,Config File -0.12.dev0,PLRATECoverageSimulation,2025-12-04 20:23,193.46772919098535,3.12.3,scripts/plm/plr_ate_config.yml +0.12.dev0,PLRATECoverageSimulation,2025-12-08 15:12,19.93783419529597,3.12.9,scripts/plm/plr_ate_config.yml diff --git a/scripts/plm/plpr_ate.py b/scripts/plm/plpr_ate.py new file mode 100644 index 0000000..a63f90b --- /dev/null +++ b/scripts/plm/plpr_ate.py @@ -0,0 +1,13 @@ +from montecover.plm import PLPRATECoverageSimulation + +# Create and run simulation with config file +sim = PLPRATECoverageSimulation( + config_file="scripts/plm/plpr_ate_config.yml", + log_level="INFO", + log_file="logs/plm/plpr_ate_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/plm/", file_prefix="plpr_ate") + +# Save config file for reproducibility +sim.save_config("results/plm/plpr_ate_config.yml") diff --git a/scripts/plm/plpr_ate_config.yml b/scripts/plm/plpr_ate_config.yml new file mode 100644 index 0000000..66a3182 --- /dev/null +++ b/scripts/plm/plpr_ate_config.yml @@ -0,0 +1,35 @@ +# Simulation parameters for PLR ATE Coverage + +simulation_parameters: + repetitions: 500 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + theta: [0.5] # Treatment effect + dim_x: [10] # Number of covariates + num_id: [1000] # number of units + num_t: [10] # number of time periods + DGP: ["dgp1", "dgp2", "dgp3"] # Data generating processes + +# Define reusable learner configurations +learner_definitions: + lasso: &lasso + name: "LassoCV" + + lgbm: &lgbm + name: "LGBM Regr." + +dml_parameters: + learners: + - ml_g: *lasso + ml_m: *lasso + - ml_g: *lgbm + ml_m: *lgbm + + score: ["partialling out", "IV-type"] + approach: ["cre_general", "cre_normal", "fd_exact", "wg_approx"] + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels diff --git a/scripts/plm/plpr_ate_tune.py b/scripts/plm/plpr_ate_tune.py new file mode 100644 index 0000000..4b63719 --- /dev/null +++ b/scripts/plm/plpr_ate_tune.py @@ -0,0 +1,13 @@ +from montecover.plm import PLPRATETuningCoverageSimulation + +# Create and run simulation with config file +sim = PLPRATETuningCoverageSimulation( + config_file="scripts/plm/plpr_ate_tune_config.yml", + log_level="INFO", + log_file="logs/plm/plpr_ate_tune_sim.log", +) +sim.run_simulation() +sim.save_results(output_path="results/plm/", file_prefix="plpr_ate_tune") + +# Save config file for reproducibility +sim.save_config("results/plm/plpr_ate_tune_config.yml") diff --git a/scripts/plm/plpr_ate_tune_config.yml b/scripts/plm/plpr_ate_tune_config.yml new file mode 100644 index 0000000..281ead8 --- /dev/null +++ b/scripts/plm/plpr_ate_tune_config.yml @@ -0,0 +1,30 @@ +# Simulation parameters for PLR ATE Coverage + +simulation_parameters: + repetitions: 100 + max_runtime: 19800 # 5.5 hours in seconds + random_seed: 42 + n_jobs: -2 + +dgp_parameters: + theta: [0.5] # Treatment effect + dim_x: [10] # Number of covariates + num_id: [1000] # number of units + num_t: [10] # number of time periods + DGP: ["dgp1", "dgp3"] # Data generating processes + +# Define reusable learner configurations +learner_definitions: + lgbm: &lgbm + name: "LGBM Regr." + +dml_parameters: + learners: + - ml_g: *lgbm + ml_m: *lgbm + + score: ["partialling out"] + approach: ["cre_general", "cre_normal", "fd_exact", "wg_approx"] + +confidence_parameters: + level: [0.95, 0.90] # Confidence levels