From c01d247af848be01e75920fe8a71d22fe0015d03 Mon Sep 17 00:00:00 2001 From: naseem173 Date: Thu, 23 Jul 2026 02:01:14 +0530 Subject: [PATCH] docs(examples): add pandera interoperability recipe --- examples/09_pandera_recipe.py | 65 +++++++++++++++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 66 insertions(+) create mode 100644 examples/09_pandera_recipe.py diff --git a/examples/09_pandera_recipe.py b/examples/09_pandera_recipe.py new file mode 100644 index 00000000..73df9667 --- /dev/null +++ b/examples/09_pandera_recipe.py @@ -0,0 +1,65 @@ +"""Recipe: validate a messy DataFrame with pandera, repair it with freshdata, +then validate again to see what's fixed and what still needs manual repair. +""" + +import pandas as pd +import pandera.pandas as pa + +import freshdata as fd + + +def main() -> None: + df = pd.DataFrame( + { + "Age": [25, 41, None, 33], + "Amount($)": ["$19.99", "$5.00", "$8.25", "n/a"], + "Status": [" shipped ", "PENDING", "shipped", "pending"], + } + ) + + raw_schema = pa.DataFrameSchema( + { + "Age": pa.Column(float, nullable=False), + "Amount($)": pa.Column(float, nullable=False), + "Status": pa.Column(str, nullable=False), + } + ) + + print("=== Step 1: validate the raw frame ===") + try: + raw_schema.validate(df) + except pa.errors.SchemaError as exc: + print(f"Validation failed, as expected:\n{exc}\n") + + cleaned = fd.clean(df) + + clean_schema = pa.DataFrameSchema( + { + "age": pa.Column(float, nullable=False), + "amount": pa.Column(float, nullable=False), + "status": pa.Column(str, pa.Check.isin(["shipped", "pending"]), nullable=False), + } + ) + + print("=== Step 2: validate the freshdata-cleaned frame ===") + try: + clean_schema.validate(cleaned) + print("Schema check passed after cleaning:") + print(cleaned) + except pa.errors.SchemaError as exc: + print(f"Still failing after cleaning (freshdata renamed/typed the columns, " + f"but doesn't impute nulls or normalize casing):\n{exc}\n") + + print("=== Step 3: repair the remaining issues, then re-validate ===") + repaired = cleaned.copy() + repaired["age"] = repaired["age"].fillna(repaired["age"].median()) + repaired["amount"] = repaired["amount"].fillna(repaired["amount"].median()) + repaired["status"] = repaired["status"].str.lower() + + clean_schema.validate(repaired) + print("Schema check passed after repair:") + print(repaired) + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md index bb687470..948591df 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,5 +19,6 @@ python examples/01_missing_values.py | [`06_large_dataset.py`](06_large_dataset.py) | Cleaning a large synthetic dataset, with timing | | [`07_pandas_integration.py`](07_pandas_integration.py) | Dropping freshdata into an existing pandas workflow | | [`08_csv_automation.py`](08_csv_automation.py) | Batch CSV cleaning automation with audit logs | +| [`09_pandera_recipe.py`](09_pandera_recipe.py) | Validating with pandera before and after freshdata cleaning | See the [documentation](https://freshcode-org.github.io/freshdata/) for full guides.