diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 31d9562..8f2fd71 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -3,6 +3,27 @@ Changelog ========= +Unreleased +---------- + +Features +'''''''' + +- :issue:`144`: ``imas convert --ids`` takes a comma-separated list of ``IDS``, + ``IDS:OCCURRENCE`` and ``IDS:*`` (all occurrences), for example + ``--ids summary,plasma_profiles:1,plasma_profiles:2``. As in the IDS path syntax, + an IDS without an occurrence means occurrence 0, so ``--ids core_profiles`` no + longer converts all occurrences: use ``--ids core_profiles:*`` for that. Unknown + IDS names, malformed occurrences and IDSs or occurrences that are not in the input + Data Entry are reported before the output Data Entry is created. + +Deprecations +'''''''''''' + +- [deprecation] :issue:`144`: the ``--occurrence`` option of ``imas convert``; + use ``--ids IDS:OCCURRENCE`` instead. To be removed in the next major release. + + What's new in IMAS-Python 2.3.1 ------------------------------- diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 9147746..907da50 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -6,9 +6,11 @@ IMAS-Python Command Line tool IMAS-Python comes with a command line tool: ``imas``. This allows you to execute some tasks without writing Python code: -- ``imas convert`` can convert Data Entries (or, optionally, single IDSs from - a Data Entry) to a different DD version. This command can also be used to - convert IDSs between different backends. +- ``imas convert`` can convert Data Entries (or, optionally, selected IDSs and + occurrences from a Data Entry, for example + ``--ids summary,plasma_profiles:1,plasma_profiles:2``) to a different DD + version. This command can also be used to convert IDSs between different + backends. - ``imas print`` can print the contents of an IDS to the terminal. - ``imas version`` shows version information of IMAS-Python. - ``imas analyze-db`` and ``imas process-db-analysis`` analyze the contents diff --git a/imas/command/cli.py b/imas/command/cli.py index da92197..e23dcfc 100644 --- a/imas/command/cli.py +++ b/imas/command/cli.py @@ -2,6 +2,8 @@ # You should have received the IMAS-Python LICENSE file with this project. """Main CLI entry point""" +from __future__ import annotations + import logging import sys from contextlib import ExitStack @@ -132,6 +134,51 @@ def _check_convert_to_plasma_ids(idss_with_occurrences): ) +ALL_OCCURRENCES = "*" + + +def _parse_ids_option( + ids: str, known_ids: list[str], dd_version: str +) -> list[tuple[str, int | str | None]]: + """Split the comma-separated --ids option into (IDS name, occurrence) pairs. + + Each entry is an IDS name, optionally followed by ``:OCCURRENCE`` or ``:*`` (all + occurrences). The occurrence is None when it is not given. Names are checked + against the target DD. + """ + entries = [] + for item in ids.split(","): + item = item.strip() + if not item: + continue + name, sep, occ = item.partition(":") + name = name.strip() + occurrence = None + if sep: + occ = occ.strip() + if occ == ALL_OCCURRENCES: + occurrence = ALL_OCCURRENCES + elif occ.isdecimal(): + occurrence = int(occ) + else: + raise click.BadParameter( + f"invalid occurrence in {item!r}: expected IDS, IDS:OCCURRENCE " + "with a non-negative integer occurrence, or IDS:*", + param_hint="'--ids'", + ) + entries.append((name, occurrence)) + entries = list(dict.fromkeys(entries)) + if not entries: + raise click.BadParameter("no IDS name given", param_hint="'--ids'") + unknown = list(dict.fromkeys(name for name, _ in entries if name not in known_ids)) + if unknown: + raise click.BadParameter( + f"unknown IDS name(s) for DD version {dd_version}: {', '.join(unknown)}", + param_hint="'--ids'", + ) + return entries + + @cli.command("convert", no_args_is_help=True) @click.argument("uri_in") @click.argument("dd_version") @@ -139,10 +186,18 @@ def _check_convert_to_plasma_ids(idss_with_occurrences): @click.option( "--ids", default="*", - help="Specify which IDS to convert. \ -If not provided, all IDSs in the data entry are converted.", + help="Specify which IDSs to convert, as a comma-separated list of IDS, " + "IDS:OCCURRENCE or IDS:* (all occurrences), for example " + "summary,plasma_profiles:1,plasma_profiles:2. An IDS without an occurrence " + "means occurrence 0. " + "If not provided, all IDSs in the data entry are converted.", +) +@click.option( + "--occurrence", + default=-1, + help="Deprecated: use --ids IDS:OCCURRENCE instead. " + "Occurrence to convert for IDSs given without one.", ) -@click.option("--occurrence", default=-1, help="Specify which occurrence to convert.") @click.option("--quiet", "-q", is_flag=True, help="Suppress progress output.") @click.option("--timeit", is_flag=True, help="Show timing information.") @click.option( @@ -166,7 +221,7 @@ def convert_ids( no_provenance: bool, convert_to_plasma_ids: bool, ): - """Convert a Data Entry (or a single IDS) to the target DD version. + """Convert a Data Entry (or selected IDSs/occurrences) to the target DD version. Provide a different backend to URI_OUT than URI_IN to convert between backends. For example: @@ -190,6 +245,22 @@ def convert_ids( else: raise UnknownDDVersion(dd_version, dd_zip.dd_xml_versions()) + # Check the requested IDS names before the output Data Entry is created + target_factory = imas.IDSFactory( + version_params.get("dd_version"), version_params.get("xml_path") + ) + if ids == "*": + ids_list = [(name, ALL_OCCURRENCES) for name in target_factory.ids_names()] + if occurrence != -1: + ids_list = [(name, occurrence) for name, _ in ids_list] + else: + ids_list = _parse_ids_option(ids, target_factory.ids_names(), dd_version) + if occurrence != -1: + logger.warning( + "--occurrence is deprecated and will be removed in the next major " + "release; use --ids IDS:OCCURRENCE instead (for example --ids summary:1)." + ) + provenance_origin_uri = "" if not no_provenance: provenance_origin_uri = uri_in @@ -197,18 +268,35 @@ def convert_ids( # Use an ExitStack to avoid three nested with-statements with ExitStack() as stack: entry_in = stack.enter_context(DBEntry(uri_in, "r")) - entry_out = stack.enter_context(DBEntry(uri_out, "x", **version_params)) # First build IDS/occurrence list so we can show a decent progress bar - ids_list = [ids] if ids != "*" else entry_out.factory.ids_names() idss_with_occurrences = [] - for ids_name in ids_list: - if occurrence == -1: - idss_with_occurrences.extend( - (ids_name, occ) for occ in entry_in.list_all_occurrences(ids_name) - ) + missing = [] + for ids_name, ids_occurrence in ids_list: + if ids_occurrence is None: # IDS given without an occurrence + ids_occurrence = occurrence if occurrence != -1 else 0 + if ids == "*" and ids_occurrence != ALL_OCCURRENCES: + # Deprecated --occurrence without --ids: keep its old behaviour + idss_with_occurrences.append((ids_name, ids_occurrence)) + continue + available = entry_in.list_all_occurrences(ids_name) + if ids_occurrence == ALL_OCCURRENCES: + idss_with_occurrences.extend((ids_name, occ) for occ in available) + if not available and ids != "*": + missing.append(f"{ids_name}:*") + elif ids_occurrence in available: + idss_with_occurrences.append((ids_name, ids_occurrence)) else: - idss_with_occurrences.append((ids_name, occurrence)) + missing.append(f"{ids_name}:{ids_occurrence}") + if missing: + # Reported before the output Data Entry is created + raise click.BadParameter( + f"not found in the input Data Entry: {', '.join(missing)}", + param_hint="'--ids'", + ) + idss_with_occurrences = list(dict.fromkeys(idss_with_occurrences)) + + entry_out = stack.enter_context(DBEntry(uri_out, "x", **version_params)) if convert_to_plasma_ids: # Sanity checks for conversion to plasma IDSs _check_convert_to_plasma_ids(idss_with_occurrences) diff --git a/imas/test/test_cli.py b/imas/test/test_cli.py index e696c05..2b821e7 100644 --- a/imas/test/test_cli.py +++ b/imas/test/test_cli.py @@ -1,4 +1,5 @@ from pathlib import Path +from unittest.mock import patch import pytest from click.testing import CliRunner @@ -135,3 +136,194 @@ def test_imas_convert_with_plasma(tmp_path): expected_comment = f"{core_edge}_{suffix} occurrence {i % 4}" ids = entry.get(f"plasma_{suffix}", i) assert ids.ids_properties.comment == expected_comment + + +@pytest.mark.cli +def test_imas_convert_ids_list(tmp_path): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + with DBEntry(f"imas:hdf5?path={in_db}", "w", dd_version="3.39.0") as entry: + for ids_name, occurrences in [ + ("core_profiles", 2), + ("equilibrium", 1), + ("summary", 1), + ("wall", 1), + ]: + ids = entry.factory.new(ids_name) + ids.ids_properties.homogeneous_time = 2 + for i in range(occurrences): + entry.put(ids, i) + + runner = CliRunner() + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + "core_profiles:*, equilibrium,core_profiles:*", + ], + ) + assert convert_result.exit_code == 0, convert_result.output + + with DBEntry(f"imas:hdf5?path={out_db}", "r", dd_version="4.1.0") as entry: + assert entry.list_all_occurrences("core_profiles") == [0, 1] + assert entry.list_all_occurrences("equilibrium") == [0] + assert entry.list_all_occurrences("summary") == [] + assert entry.list_all_occurrences("wall") == [] + + +@pytest.mark.cli +def test_imas_convert_unknown_ids(tmp_path): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + with DBEntry(f"imas:hdf5?path={in_db}", "w", dd_version="3.39.0") as entry: + ids = entry.factory.core_profiles() + ids.ids_properties.homogeneous_time = 2 + entry.put(ids) + + runner = CliRunner() + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + "core_profiles,core_profile", + ], + ) + # A misspelled name is reported before anything is converted or written + assert convert_result.exit_code == 2 + assert "unknown IDS name(s) for DD version 4.1.0: core_profile" in ( + convert_result.output + ) + assert not out_db.exists() + + +def _write_occurrences(db_path, idss): + with DBEntry(f"imas:hdf5?path={db_path}", "w", dd_version="3.39.0") as entry: + for ids_name, occurrences in idss: + ids = entry.factory.new(ids_name) + ids.ids_properties.homogeneous_time = 2 + for i in range(occurrences): + entry.put(ids, i) + + +@pytest.mark.cli +def test_imas_convert_ids_with_occurrences(tmp_path): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + _write_occurrences(in_db, [("core_profiles", 3), ("equilibrium", 2), ("wall", 1)]) + + runner = CliRunner() + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + "equilibrium, core_profiles:1,core_profiles:2,core_profiles:1,wall:*", + ], + ) + assert convert_result.exit_code == 0, convert_result.output + + with DBEntry(f"imas:hdf5?path={out_db}", "r", dd_version="4.1.0") as entry: + # As in the IDS path syntax, an IDS without an occurrence means occurrence 0 + assert entry.list_all_occurrences("equilibrium") == [0] + assert entry.list_all_occurrences("core_profiles") == [1, 2] + assert entry.list_all_occurrences("wall") == [0] + + +@pytest.mark.cli +def test_imas_convert_occurrence_option_is_deprecated(tmp_path): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + _write_occurrences(in_db, [("core_profiles", 3), ("equilibrium", 2)]) + + runner = CliRunner() + with patch("imas.command.cli.logger") as logger: + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + "core_profiles,equilibrium:0", + "--occurrence", + "2", + ], + ) + assert convert_result.exit_code == 0, convert_result.output + assert any("deprecated" in str(c.args[0]) for c in logger.warning.call_args_list) + + with DBEntry(f"imas:hdf5?path={out_db}", "r", dd_version="4.1.0") as entry: + # --occurrence still applies to IDSs given without an occurrence + assert entry.list_all_occurrences("core_profiles") == [2] + assert entry.list_all_occurrences("equilibrium") == [0] + + +@pytest.mark.cli +@pytest.mark.parametrize( + "ids", + ["core_profiles:x", "core_profiles:-1", "core_profiles:", "core_profiles:\u00b2"], +) +def test_imas_convert_invalid_occurrence(tmp_path, ids): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + _write_occurrences(in_db, [("core_profiles", 1)]) + + runner = CliRunner() + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + ids, + ], + ) + assert convert_result.exit_code == 2 + assert "invalid occurrence" in convert_result.output + assert not out_db.exists() + + +@pytest.mark.cli +@pytest.mark.parametrize( + "ids, missing", + [ + ("core_profiles:3", "core_profiles:3"), + ("core_profiles,wall", "wall:0"), + ("core_profiles:*,wall:*", "wall:*"), + ], +) +def test_imas_convert_ids_missing_from_input(tmp_path, ids, missing): + in_db = tmp_path / "in" + out_db = tmp_path / "out" + _write_occurrences(in_db, [("core_profiles", 2)]) + + runner = CliRunner() + with runner.isolated_filesystem(tmp_path): + convert_result = runner.invoke( + convert_ids, + [ + f"imas:hdf5?path={in_db}", + "4.1.0", + f"imas:hdf5?path={out_db}", + "--ids", + ids, + ], + ) + # Reported before the output Data Entry is created + assert convert_result.exit_code == 2 + assert f"not found in the input Data Entry: {missing}" in convert_result.output + assert not out_db.exists()