Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/452.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allow bundle consumers to download and verify certified datasets and write a receipt without installing Python packages.
9 changes: 8 additions & 1 deletion docs/bundles.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,21 @@ Managed datasets are downloaded from the Hugging Face artifact specified in the
bundle. GCS dataset URIs are unsupported. The separate UK geography lookup files
retain their existing storage implementation.

Country-specific and package-only installs are supported:
Country-specific, package-only, and dataset-only installs are supported:

```bash
uvx --from policyengine policyengine bundle install --country uk
uvx --from policyengine policyengine bundle install --no-datasets
uvx --from policyengine policyengine bundle install --no-packages --yes
```

Use `--yes` for CI/CD. Without `--yes`, dataset downloads ask for confirmation.
Use `--no-packages` when a lockfile or another package manager owns the Python
environment. It downloads and verifies the selected certified datasets and
writes the bundle receipt without resolving Python, creating a virtual
environment, or installing packages. Dataset-only receipts omit
`target_python`; `bundle status` therefore inspects its current interpreter
unless `--python` or `--venv` is supplied explicitly.

The canonical bundle manifest is `src/policyengine/data/bundle/manifest.json`.
Derived artifacts are:
Expand Down
8 changes: 8 additions & 0 deletions docs/release-bundles.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,17 @@ Useful variants:
uvx --from policyengine policyengine bundle install
uvx --from policyengine policyengine bundle install --country uk
uvx --from policyengine policyengine bundle install --no-datasets
uvx --from policyengine policyengine bundle install --no-packages --yes
uvx --from policyengine policyengine bundle install --yes
```

Use `--no-packages` when packages are installed from a separate lockfile. The
command still downloads and verifies the selected certified datasets and writes
the bundle receipt, but it does not resolve Python, create a virtual environment,
or install packages. Dataset-only receipts omit `target_python`; `bundle status`
therefore inspects its current interpreter unless `--python` or `--venv` is
supplied explicitly.

Check a local environment against a bundle:

```bash
Expand Down
30 changes: 22 additions & 8 deletions src/policyengine/bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,18 +394,32 @@ def install_bundle(
countries: Optional[Sequence[str]] = None,
data_dir: Path = DEFAULT_DATA_DIR,
no_datasets: bool = False,
no_packages: bool = False,
yes: bool = False,
dry_run: bool = False,
) -> dict[str, Any]:
if no_datasets and no_packages:
raise BundleError("Pass either no_datasets or no_packages, not both.")
if no_packages and (python is not None or venv is not None):
raise BundleError(
"Python and virtualenv targets cannot be used when packages are skipped."
)

manifest = load_bundle_manifest(version, manifest_ref=manifest_ref)
selected_countries = normalise_countries(countries, manifest)
requirements = bundle_install_requirements(manifest, countries=selected_countries)
target_python = resolve_target_python(
python=python,
venv=venv,
create_venv=not dry_run,
)
install_package_scaffold(target_python, requirements, dry_run=dry_run)
requirements: list[str] = []
target_python: Optional[Path] = None
if not no_packages:
requirements = bundle_install_requirements(
manifest,
countries=selected_countries,
)
target_python = resolve_target_python(
python=python,
venv=venv,
create_venv=not dry_run,
)
install_package_scaffold(target_python, requirements, dry_run=dry_run)
installed_datasets: list[dict[str, Any]] = []
if not no_datasets:
dataset_entries = _selected_dataset_plans(
Expand Down Expand Up @@ -445,7 +459,7 @@ def install_bundle(
"countries": selected_countries,
"datasets": installed_datasets,
"data_dir": str(data_dir),
"target_python": str(target_python),
"target_python": str(target_python) if target_python is not None else None,
}


Expand Down
9 changes: 8 additions & 1 deletion src/policyengine/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,17 @@ def _parser() -> argparse.ArgumentParser:
choices=("us", "uk"),
help="Country to include. Repeat for multiple countries. Defaults to all.",
)
bundle_install.add_argument(
bundle_install_components = bundle_install.add_mutually_exclusive_group()
bundle_install_components.add_argument(
"--no-datasets",
action="store_true",
help="Install packages without downloading certified datasets.",
)
bundle_install_components.add_argument(
"--no-packages",
action="store_true",
help="Download certified datasets without installing packages.",
)
bundle_install.add_argument(
"--data-dir",
type=Path,
Expand Down Expand Up @@ -387,6 +393,7 @@ def _install_bundle(args: argparse.Namespace) -> int:
countries=args.country,
data_dir=args.data_dir,
no_datasets=args.no_datasets,
no_packages=args.no_packages,
yes=args.yes,
dry_run=args.dry_run,
)
Expand Down
83 changes: 83 additions & 0 deletions tests/test_bundle.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,69 @@ def fake_materialize(plan):
assert receipt["datasets"] == result["datasets"]


def test_install_bundle_datasets_only_skips_python_and_packages(monkeypatch, tmp_path):
calls = []

def fail_if_called(*args, **kwargs):
pytest.fail("Datasets-only installation must not manage Python packages.")

def fake_materialize(plan):
calls.append(plan)
plan.destination.parent.mkdir(parents=True, exist_ok=True)
plan.destination.write_bytes(b"materialized")
return MaterializedDataset(
data_package_name=plan.data_package_name,
repo_type=plan.repo_type,
revision=plan.revision,
source_uri=plan.source_uri,
sha256=plan.sha256,
path=plan.destination,
)

monkeypatch.setattr(bundle, "resolve_target_python", fail_if_called)
monkeypatch.setattr(bundle, "install_package_scaffold", fail_if_called)
monkeypatch.setattr(bundle, "_reuse_or_download_bundle_files", fake_materialize)

result = bundle.install_bundle(
countries=["uk"],
data_dir=tmp_path,
no_packages=True,
yes=True,
)

assert [plan.country_id for plan in calls] == ["uk"]
assert result["requirements"] == []
assert result["target_python"] is None
receipt = bundle.read_receipt(tmp_path)
assert receipt is not None
assert "target_python" not in receipt
assert receipt["datasets"] == result["datasets"]


def test_install_bundle_rejects_skipping_all_components(tmp_path):
with pytest.raises(
bundle.BundleError,
match="Pass either no_datasets or no_packages, not both",
):
bundle.install_bundle(
data_dir=tmp_path,
no_datasets=True,
no_packages=True,
)


def test_install_bundle_rejects_python_target_when_packages_are_skipped(tmp_path):
with pytest.raises(
bundle.BundleError,
match="Python and virtualenv targets cannot be used",
):
bundle.install_bundle(
python=sys.executable,
data_dir=tmp_path,
no_packages=True,
)


def test_status_matches_receipt_and_packages(monkeypatch, tmp_path):
manifest = _manifest_with_dataset_sha("uk", _sha256(b"data"))
datasets = [
Expand Down Expand Up @@ -377,6 +440,26 @@ def test_bundle_install_dry_run_cli_uses_standard_flags(capsys):
)


def test_bundle_install_datasets_only_dry_run_cli_skips_packages(capsys):
exit_code = cli_main(
[
"bundle",
"install",
"--country",
"uk",
"--no-packages",
"--yes",
"--dry-run",
]
)

assert exit_code == 0
output = capsys.readouterr().out
assert "pip install" not in output
assert "download " in output
assert '"target_python": null' in output


def test_bundle_verify_cli_handles_unknown_bundle(capsys):
exit_code = cli_main(["bundle", "verify", "0.0.0"])

Expand Down
Loading