diff --git a/README.md b/README.md index 0030c54..e9ca5ce 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@staging" ``` # staging's latest release (early access) — pin to the tag the "staging" badge above shows -pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.34-staging" +pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.35-staging" ``` ## Usage @@ -305,11 +305,15 @@ endpoints (`dds_documents/client.py`) are still assumptions to confirm with the ## Notebook magics for SDI and DDS (`%sdi` / `%dds`) -`%sdi` (`SdiSession`) reads SDI and `%dds` (`DdsSession`) pushes files to DDS. Both are built -on first use from the kernel environment like `%catalog`'s and `%ingest`'s sessions. `%dds -push` puts files in a folder of the dataset `%catalog use(...)` last selected; `path=` picks -another one, absolute or relative to that context, as any `%catalog` path. The folder names -can be typed bare on a `%dds` line (worked example: `debugger/sdi_session_example.ipynb`): +`%sdi` (`SdiSession`) reads SDI and `%dds` (`DdsSession`) pushes files to DDS, lists them and +gets them back. Both are built on first use from the kernel environment like `%catalog`'s and +`%ingest`'s sessions. `%dds push` puts files in a folder of the dataset `%catalog use(...)` last +selected, `%dds list` shows the file names already in one, and `%dds get` downloads one of them +into the notebook's folder (`output_dir=` for another; a missing file is an error). `path=` +picks another dataset, absolute or relative to that context, as any `%catalog` path. The folder +is compulsory and is one of DDS's eight document folders, typed bare on a `%dds` line; a +misspelt, quoted or missing folder is an error that lists the valid ones (worked example: +`debugger/sdi_session_example.ipynb`): ```python import eea_datalakehouse.notebook # registers %catalog/%ingest/%sdi/%dds @@ -318,6 +322,9 @@ import eea_datalakehouse.notebook # registers %catalog/%ingest/%sdi/%dds %dds push(DELIVERABLES, "report.pdf", "annex/*.xlsx") record = %sdi get_xml("070d9baa-448d-4168-8514-7dadb3ad876d") %dds push(METADATA, record, overwrite=False) +%dds list(DELIVERABLES) # report.pdf and the annexes, sorted +%dds list(DELIVERABLES, "*.pdf") # filter: a wildcard, ".xlsx", "report", or a list of them +f = %dds get(DELIVERABLES, "report.pdf") # -> ./report.pdf; f is the local file %dds target(ASSESSMENTS) # where a push would go %dds help ``` @@ -510,7 +517,7 @@ context itself — raises `CatalogSessionError` if none is set yet: | `dds_ingestion/progress.py` | tqdm progress bar with graceful fallback | | `dds_ingestion/folder.py` | `FolderIngest` orchestration (scan/parallel/resume) | | `dds_documents/client.py` | `DocumentsClient` — put/get/list plain files in DDS folders (no ingest) | -| `dds_documents/controller.py` | `DdsController` / `DdsFolder` — push files to a catalog node's DDS document folders (metadata, deliverables, ...) | +| `dds_documents/controller.py` | `DdsController` / `DdsFolder` — push, list and get files in a catalog node's DDS document folders (metadata, deliverables, ...) | | `dds_documents/paths.py` | catalog (`a.b.c`) ⇄ storage (`a/b/c`) path conversion | | `dds_documents/session.py` | `DdsSession` — what `%dds` dispatches onto | | `geopackage/controller.py` | `GpkgController` — Dremio tables/views to a GeoPackage, streamed over parallel Flight channels | @@ -602,10 +609,10 @@ To pin to one specific release instead, use the exact tag the live badges under [Releasing a new version](#releasing-a-new-version)): ```python -%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.34" +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.35" # staging's latest release (early access) -%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.34-staging" +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.35-staging" ``` Use the `%pip` magic rather than `!pip` — it installs into the kernel the diff --git a/debugger/test_catalog_magic.ipynb b/debugger/test_catalog_magic.ipynb index 61adec3..298216e 100644 --- a/debugger/test_catalog_magic.ipynb +++ b/debugger/test_catalog_magic.ipynb @@ -107,18 +107,66 @@ "name": "stdout", "output_type": "stream", "text": [ - "%dds methods — usage: %dds push(DELIVERABLES, \"report.pdf\") | %dds push(METADATA, [\"a.xml\", \"b.xml\"]) (into the %catalog use(...) folder)\n", + "%dds methods — usage: %dds push(DELIVERABLES, \"report.pdf\") | %dds push(METADATA, \"a.xml\", \"b.xml\") | %dds list(DELIVERABLES) | %dds get(DELIVERABLES, \"report.pdf\") (in the %catalog use(...) folder)\n", "Files go to //, e.g. after\n", " %catalog use(\"catalog.water_management_resources.bathing_water.bwd\")\n", " %dds push(DELIVERABLES, \"report.pdf\")\n", "-> catalog/water_management_resources/bathing_water/bwd/deliverables/report.pdf\n", + "\n", + "push examples:\n", + " files — one or more, one after another, as a list, or as a glob:\n", + " %dds push(DELIVERABLES, \"report.pdf\", \"annex.xlsx\")\n", + " %dds push(DELIVERABLES, [\"report.pdf\", \"annex.xlsx\"])\n", + " %dds push(ANALYSES, \"results/*.csv\")\n", + " a GeoPackage from %gpkg export (the report is the file):\n", + " report = %gpkg export()\n", + " %dds push(DELIVERABLES, report)\n", + " an SDI record from %sdi get_xml (stored as .xml):\n", + " record = %sdi get_xml(\"070d9baa-448d-4168-8514-7dadb3ad876d\")\n", + " %dds push(METADATA, record)\n", + " folder — compulsory, one of DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES,\n", + " REFERENCES, DELIVERABLES, OTHERS, typed bare (a string like \"methodology\" is refused):\n", + " %dds push(METHODOLOGY, \"method.docx\")\n", + " path= — another dataset than the %catalog context, absolute or relative to it:\n", + " %dds push(DELIVERABLES, \"report.pdf\", path=\"catalog.water_management_resources.wfd\")\n", + " %dds push(DELIVERABLES, \"report.pdf\", path=\"../wfd\")\n", + " overwrite= — True (default) replaces a file already there;\n", + " False uploads nothing if any is:\n", + " %dds push(METADATA, record, overwrite=False)\n", + " target_name= — another name in DDS: a string for one file, a dict for several\n", + " (files left out keep their name):\n", + " %dds push(METADATA, record, target_name=\"bwd_2025.xml\")\n", + " %dds push(DELIVERABLES, \"draft_v3.pdf\", \"annex.xlsx\", target_name={\"draft_v3.pdf\": \"report_2025.pdf\"})\n", + " all together:\n", + " %dds push(DELIVERABLES, \"draft_v3.pdf\", \"annex/*.xlsx\", path=\"../wfd\", overwrite=False, target_name={\"draft_v3.pdf\": \"report_2025.pdf\"})\n", + " (a %dds call is one line — it cannot continue on the next one)\n", + " check first, without uploading: %dds target(DELIVERABLES) or %dds target(DELIVERABLES, path=\"../wfd\")\n", + "\n", + "list examples — folder is compulsory, path= as for push:\n", + " %dds list(DELIVERABLES)\n", + " %dds list(METADATA, path=\"../wfd\")\n", + " names = %dds list(ASSESSMENTS)\n", + " pattern — optional, ignores case; a wildcard, an extension, text in the name, or a list:\n", + " %dds list(DELIVERABLES, \"*.pdf\")\n", + " %dds list(DELIVERABLES, \".xlsx\")\n", + " %dds list(DELIVERABLES, \"report\")\n", + " %dds list(DELIVERABLES, \"report_202?.*\")\n", + " %dds list(DELIVERABLES, [\"*.pdf\", \"*.docx\"], path=\"../wfd\")\n", + "\n", + "get examples — folder and filename are compulsory, path= as for push:\n", + " %dds get(DELIVERABLES, \"report.pdf\") # -> ./report.pdf\n", + " f = %dds get(METADATA, \"bwd.xml\", path=\"../wfd\", output_dir=\"downloads\")\n", + " %dds get(DELIVERABLES, \"annex/a.xlsx\", overwrite=False)\n", + "\n", + "A folder that is not one of the eight (DELIVERABLE, \"deliverables\", left out) is an error that lists the valid ones.\n", + "\n", "Environment: DDS_BASE_URL — read from the first .env found in the notebook's folder or a parent when %dds starts, else from the kernel environment — and a Dremio identity: _DREMIO_USER/_DREMIO_PWD (as %ingest) or DREMIO_USERNAME/DREMIO_TOKEN (as %catalog).\n" ] }, { "data": { "text/html": [ - "
CommandParametersDescription
pushfolder, files, path=None, overwrite=TrueUpload files to path/folder/ in DDS with PUT, one request per file — a document upload, never an ingest. path defaults to the %catalog use(...) context and is resolved like any %catalog path (absolute, or relative to the context). folder is one of DDS's document folders: DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES, REFERENCES, DELIVERABLES, OTHERS (typed bare, or as a string in any case), or one a DDS admin added, as a string — checked against DDS's live list. files is one file or a list: a local path (a glob like "annex/*.pdf" too), or a %sdi get_xml record (stored as <uuid>.xml). Every file is read and checked before the first upload. overwrite=True (default) replaces a file already there; overwrite=False uploads nothing if any is.
targetfolder, path=NoneShow the DDS folder push would upload to, without uploading anything.
foldersList DDS's document folders now — the built-in ones plus any a DDS admin added.
dds_base_urlShow the DDS URL push uses, and where it came from.
" + "
CommandParametersDescription
pushfolder, files..., path=None, overwrite=True, target_name=''Upload files to path/folder/ in DDS with PUT, one request per file — a document upload, never an ingest. path defaults to the %catalog use(...) context and is resolved like any %catalog path (absolute, or relative to the context). folder is compulsory and must be one of DDS's document folders, typed bare (no quotes): DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES, REFERENCES, DELIVERABLES, OTHERS. files are one or more, given one after another or as a list: local paths (globs like "annex/*.pdf" too), %sdi get_xml records (stored as <uuid>.xml) or %gpkg export reports (the .gpkg written). Every file is read and checked before the first upload. target_name renames files in DDS: a string for a single file (target_name="bwd_2025.xml"), or a dict from each file's own name to its new name (target_name={"draft_v3.pdf": "report_2025.pdf"}); files left out, or an empty target_name, keep their own name. overwrite=True (default) replaces a file already there; overwrite=False uploads nothing if any is.
listfolder, pattern=None, path=NoneShow the names of the files in path/folder/ in DDS, sorted. folder is compulsory, as for push; path defaults to the %catalog use(...) context. pattern (optional) keeps only the names that match it, ignoring case: a wildcard ("*.pdf", "report_202?.*", "[ab]*"), an extension (".xlsx") or text in the name ("report"); a list keeps the names matching any (["*.pdf", "*.docx"]). The result iterates like a list of names, so `"report.pdf" in %dds list(DELIVERABLES)` works.
getfolder, filename, path=None, output_dir=None, overwrite=TrueDownload one file from path/folder/ in DDS and return the local file. folder and filename are compulsory (filename as list shows it, e.g. "annex/a.xlsx" for a subfolder); path defaults to the %catalog use(...) context. The file is saved in output_dir (default: the notebook's folder) under its own name; overwrite=True (default) replaces a local file of that name, overwrite=False refuses. A file DDS does not have there is an error. The result is the local path, so open(f), pd.read_csv(f) or %dds push(OTHERS, f) take it as is.
targetfolder, path=NoneShow the DDS folder push would upload to, without uploading anything.
foldersList the folders push, list and get accept.
dds_base_urlShow the DDS URL push uses, and where it came from.
" ], "text/plain": [ "" @@ -1828,7 +1876,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "228ebbc7", "metadata": {}, "outputs": [ @@ -1846,7 +1894,17 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 12, + "id": "5d690288", + "metadata": {}, + "outputs": [], + "source": [ + "aa= %dds list(METADATA)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "id": "6b76127e", "metadata": {}, "outputs": [ @@ -1855,10 +1913,10 @@ "output_type": "stream", "text": [ "exporting 3 table(s)/view(s) from catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft to /home/esparza/EEADatahub/EEALakeHouse.python/debugger/output.gpkg, 3 at a time\n", - "✓ 1/3 designation_types ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.designation_types: attributes, 100 rows, 0.5s\n", - "✓ 2/3 designated_areas ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.designated_areas: attributes, 100 rows, 0.5s\n", - "✓ 3/3 protected_site_polygon ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.protected_site_polygon: features, MULTIPOLYGON, SRS 4326, 100 rows, 1.7s\n", - "done: /home/esparza/EEADatahub/EEALakeHouse.python/debugger/output.gpkg — 3 layer(s), 300 rows in 1.7s\n" + "✓ 1/3 designation_types ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.designation_types: attributes, 100 rows, 0.9s\n", + "✓ 2/3 designated_areas ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.designated_areas: attributes, 100 rows, 1.0s\n", + "✓ 3/3 protected_site_polygon ← catalog.biodiversity_ecosystems.protected_areas.dfl_natda.draft.protected_site_polygon: features, MULTIPOLYGON, SRS 4326, 100 rows, 2.7s\n", + "done: /home/esparza/EEADatahub/EEALakeHouse.python/debugger/output.gpkg — 3 layer(s), 300 rows in 2.7s\n" ] } ], @@ -1868,7 +1926,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "id": "8cc55f42", "metadata": {}, "outputs": [ @@ -1879,7 +1937,7 @@ " catalog/biodiversity_ecosystems/protected_areas/dfl_natda/draft/deliverables/output.gpkg" ] }, - "execution_count": 6, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1887,6 +1945,28 @@ "source": [ "%dds push(DELIVERABLES, result, overwrite=True)" ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "82afbc85", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "got catalog/biodiversity_ecosystems/protected_areas/dfl_natda/draft/deliverables/output.gpkg (860.0 KB) -> /home/esparza/EEADatahub/EEALakeHouse.python/debugger/output.gpkg" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result=%dds get(DELIVERABLES, \"output.gpkg\")\n", + "result" + ] } ], "metadata": { diff --git a/pyproject.toml b/pyproject.toml index 2dcf056..7296b2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "EEADataLakehouse" -version = "0.1.34" +version = "0.1.35" description = "Data preparation and validation utilities for the EEA data lakehouse pipeline." readme = "README.md" requires-python = ">=3.11" diff --git a/src/eea_datalakehouse/dds_documents/__init__.py b/src/eea_datalakehouse/dds_documents/__init__.py index a2295dd..6dae0bc 100644 --- a/src/eea_datalakehouse/dds_documents/__init__.py +++ b/src/eea_datalakehouse/dds_documents/__init__.py @@ -7,6 +7,8 @@ with DdsController.from_env() as dds: dds.push("catalog.water.bwd", DdsFolder.DELIVERABLES, ["report.pdf"]) + dds.list("catalog.water.bwd", DdsFolder.DELIVERABLES) + report = dds.get("catalog.water.bwd", DdsFolder.DELIVERABLES, "report.pdf") :class:`DocumentsClient` is the HTTP layer underneath, one method per DDS endpoint; :class:`DdsSession` is what the `%dds` magic dispatches onto. @@ -18,20 +20,26 @@ from .controller import ( DdsController, DdsFile, + DdsFileNotFoundError, DdsFilesExistError, DdsFolder, + DdsListing, DdsPushError, PushReport, folder_name, + valid_folders_hint, ) from .paths import path_segments, to_catalog_path, to_storage_path -from .session import DdsSession, DdsSessionError +from .session import DdsDownload, DdsSession, DdsSessionError __all__ = [ "DdsController", + "DdsDownload", "DdsFile", + "DdsFileNotFoundError", "DdsFilesExistError", "DdsFolder", + "DdsListing", "DdsPushError", "DdsSession", "DdsSessionError", @@ -43,4 +51,5 @@ "path_segments", "to_catalog_path", "to_storage_path", + "valid_folders_hint", ] diff --git a/src/eea_datalakehouse/dds_documents/client.py b/src/eea_datalakehouse/dds_documents/client.py index e4f2817..d310646 100644 --- a/src/eea_datalakehouse/dds_documents/client.py +++ b/src/eea_datalakehouse/dds_documents/client.py @@ -10,17 +10,19 @@ ================================ ============================================ :meth:`DocumentsClient.put` ``PUT /api/v1/files/{path}`` :meth:`~DocumentsClient.get` ``GET /api/v1/files/{path}`` -:meth:`~DocumentsClient.exists` ``GET /api/v1/files/{path}`` -:meth:`~DocumentsClient.list` ``GET /api/v1/files?prefix={folder}`` +:meth:`~DocumentsClient.exists` ``HEAD /api/v1/files/{path}`` +:meth:`~DocumentsClient.list` ``GET /api/v1/files/{folder}/`` (paged by ``cursor``) :meth:`~DocumentsClient.folders` ``GET /api/v1/folders`` ================================ ============================================ -``PUT`` and ``GET /api/v1/folders`` follow the Dremio Document Service's own -routes: the upload body is the raw file, and overwriting is asked for with the -``X-DDS-Overwrite: true`` header (an existing file otherwise answers 409). The -other shapes are still **interface assumptions** to confirm with the DDS team -(see ``docs/sdi-integration-plan.md``). Each one is a module constant so a -confirmed contract is a one-line change. +These are the Dremio Document Service's own routes (its ``/openapi.json``): +the upload body is the raw file, and overwriting is asked for with the +``X-DDS-Overwrite: true`` header (an existing file otherwise answers 409). +``GET /api/v1/files/{path}`` downloads a file, or — when the path ends in +``/`` — lists that folder, a page (``limit``, default 200) at a time. The +service does not publish the listing's JSON shape, so :meth:`list` reads the +usual ones (see :func:`_listing_entries`) and says which keys it found when a +response is none of them. """ from __future__ import annotations @@ -34,11 +36,20 @@ from ..dds_ingestion.credentials import DremioCreds, load_base_url, load_creds DEFAULT_TIMEOUT = 60.0 -# Assumed DDS document contract — to confirm with the DDS team. FILES_ENDPOINT = "/api/v1/files/{path}" -LIST_ENDPOINT = "/api/v1/files" +# A trailing slash asks DDS for the folder's listing instead of a download; +# without it, /api/v1/files redirects (307) to /api/v1/files/. +LIST_ENDPOINT = "/api/v1/files/{path}/" FOLDERS_ENDPOINT = "/api/v1/folders" OVERWRITE_HEADER = "X-DDS-Overwrite" +LIST_PAGE_SIZE = 200 +# A listing longer than this many pages is a server that keeps handing back a cursor. +_MAX_LIST_PAGES = 10_000 +# Where a listing page keeps its entries, its next-page cursor, and an entry its path. +_ENTRY_KEYS = ("items", "entries", "files", "children", "objects", "contents", "results", "data") +_CURSOR_KEYS = ("next_cursor", "nextCursor", "cursor", "next") +_PATH_KEYS = ("path", "key", "rel_path", "name", "filename") +_FOLDER_TYPES = {"dir", "directory", "folder", "prefix", "collection"} # Error bodies are quoted back to the caller; cap them so an HTML error page # does not bury the status line it came with. _MAX_ERROR_CHARS = 500 @@ -146,21 +157,44 @@ def get(self, path: str) -> bytes | None: return resp.content def exists(self, path: str) -> bool: - """Whether a document is stored at ``path``.""" - return self.get(path) is not None + """Whether a document is stored at ``path`` (a ``HEAD``: nothing is downloaded).""" + endpoint = self._file_endpoint(path) + resp = self._http.head(f"{self._base_url}{endpoint}", headers=self._auth_headers) + if resp.status_code == 404: + return False + _raise_for_status(resp, f"HEAD {endpoint}") + return True def list(self, folder: str) -> list[str]: - """Paths of the documents under ``folder``.""" - prefix = _clean_path(folder) - resp = self._http.get( - f"{self._base_url}{LIST_ENDPOINT}", - params={"prefix": prefix}, - headers=self._auth_headers, + """The files in ``folder``, as DDS names them (a path or a bare name), + every page of them; subfolders are left out. Empty if DDS has no such + folder (404).""" + endpoint = LIST_ENDPOINT.format(path=quote(_clean_path(folder), safe="/")) + params: dict[str, str | int] = {"limit": LIST_PAGE_SIZE} + found: list[str] = [] + for _ in range(_MAX_LIST_PAGES): + resp = self._http.get( + f"{self._base_url}{endpoint}", params=params, headers=self._auth_headers + ) + if resp.status_code == 404: + return found + _raise_for_status(resp, f"GET {endpoint}") + try: + payload: Any = resp.json() + except ValueError as exc: + raise DocumentsApiError( + resp.status_code, + f"expected a JSON folder listing, got {resp.headers.get('content-type')!r}", + where=f"GET {endpoint}", + ) from exc + entries, cursor = _listing_entries(payload, where=f"GET {endpoint}") + found.extend(_entry_path(entry) for entry in entries if not _is_folder_entry(entry)) + if not cursor or cursor == params.get("cursor"): + return found + params["cursor"] = cursor + raise DocumentsApiError( + 200, f"listing did not end after {_MAX_LIST_PAGES} pages", where=f"GET {endpoint}" ) - _raise_for_status(resp, f"GET {LIST_ENDPOINT}?prefix={prefix}") - payload: Any = resp.json() - rows = payload.get("files", []) if isinstance(payload, dict) else payload - return [row["path"] if isinstance(row, dict) else str(row) for row in rows] def folders(self) -> list[str]: """The document folder names DDS shows under every catalog node. @@ -190,6 +224,43 @@ def _clean_path(path: str) -> str: return cleaned +def _listing_entries(payload: Any, *, where: str) -> tuple[list[Any], str | None]: + """``(entries, next cursor)`` of one listing page: a bare list, or an object + keeping its entries under one of ``_ENTRY_KEYS`` and the next page's cursor + under one of ``_CURSOR_KEYS`` (or none on the last page).""" + if isinstance(payload, list): + return payload, None + if isinstance(payload, dict): + key = next((k for k in _ENTRY_KEYS if isinstance(payload.get(k), list)), None) + if key is not None: + cursor = next((payload[k] for k in _CURSOR_KEYS if payload.get(k)), None) + return payload[key], str(cursor) if cursor is not None else None + keys = ", ".join(map(str, payload)) if isinstance(payload, dict) else type(payload).__name__ + raise DocumentsApiError( + 200, f"unrecognised folder listing (top-level keys: {keys or 'none'})", where=where + ) + + +def _entry_path(entry: Any) -> str: + if isinstance(entry, dict): + for key in _PATH_KEYS: + if isinstance(entry.get(key), str) and entry[key]: + return str(entry[key]) + raise DocumentsApiError( + 200, f"listing entry without a path or name (keys: {', '.join(map(str, entry))})" + ) + return str(entry) + + +def _is_folder_entry(entry: Any) -> bool: + if isinstance(entry, dict): + kind = str(entry.get("type") or entry.get("kind") or "").lower() + if kind in _FOLDER_TYPES or entry.get("is_dir") is True or entry.get("isDir") is True: + return True + return any(isinstance(entry.get(k), str) and entry[k].endswith("/") for k in _PATH_KEYS) + return str(entry).endswith("/") + + def _error_message(resp: httpx.Response) -> str: """The ``message`` of a DDS ``{error, message}`` body, else the capped text.""" message = resp.text diff --git a/src/eea_datalakehouse/dds_documents/controller.py b/src/eea_datalakehouse/dds_documents/controller.py index 859d70a..89a8e14 100644 --- a/src/eea_datalakehouse/dds_documents/controller.py +++ b/src/eea_datalakehouse/dds_documents/controller.py @@ -2,17 +2,26 @@ Every catalog node in DDS (``catalog/water/bwd``) shows a set of document folders for files that are not data — metadata records, methodology, -assessments, deliverables, ... They are not Dremio entities. The target -folder is always one of :class:`DdsFolder`, DDS's eight built-in folders; -free-text folder names are refused. -:class:`DdsController` uploads files there with ``PUT``, one request per file:: +assessments, deliverables, ... They are not Dremio entities. The folder is +always one of :class:`DdsFolder`, DDS's eight built-in folders; free-text +folder names are refused, and the error names the valid ones. +:class:`DdsController` uploads files there with ``PUT``, one request per file, +lists what a folder holds and fetches a file back:: with DdsController.from_env() as dds: dds.push("catalog.water.bwd", DdsFolder.DELIVERABLES, ["report.pdf", "annex.xlsx"]) # -> catalog/water/bwd/deliverables/report.pdf, .../annex.xlsx + dds.list("catalog.water.bwd", DdsFolder.DELIVERABLES) + # -> 2 file(s) in catalog/water/bwd/deliverables: annex.xlsx, report.pdf + dds.list("catalog.water.bwd", DdsFolder.DELIVERABLES, "*.pdf") + # -> 1 file(s) matching '*.pdf' in catalog/water/bwd/deliverables: report.pdf + dds.get("catalog.water.bwd", DdsFolder.DELIVERABLES, "report.pdf") + # -> DdsFile(name='report.pdf', data=<... bytes>) ================================= ============================================== :meth:`DdsController.push` upload files to ``{dds_path}/{folder}/{name}`` +:meth:`DdsController.list` the names of the files in ``{dds_path}/{folder}`` +:meth:`DdsController.get` one file of ``{dds_path}/{folder}``, as a :class:`DdsFile` :meth:`DdsController.target` the folder they would go to, without uploading ================================= ============================================== @@ -22,11 +31,12 @@ from __future__ import annotations +import fnmatch import glob import mimetypes import os from collections import Counter -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path @@ -55,20 +65,40 @@ class DdsFolder(StrEnum): DELIVERABLES = "deliverables" OTHERS = "others" + @classmethod + def _missing_(cls, value: object) -> DdsFolder: + # DdsFolder("deliverable"): say which values exist, not just that this one doesn't. + raise ValueError(f"{value!r} is not a DDS folder; {valid_folders_hint()}") + + +def valid_folders_hint() -> str: + """The clause every folder error carries: which values are valid.""" + return "valid folders are " + ", ".join(member.name for member in DdsFolder) + + +def closest_folder(text: str) -> DdsFolder | None: + """The :class:`DdsFolder` ``text`` most likely meant (any case, singular or + plural, a leading/trailing slash), or ``None``.""" + key = text.strip().strip("/").lower() + for member in DdsFolder: + if key in (member.value, member.value.removesuffix("s")): + return member + return None + def folder_name(value: DdsFolder) -> str: """``value``'s folder name in DDS. ``value`` must be a :class:`DdsFolder`: - the target folder is always one of the enumeration, never free text.""" + the folder is always one of the enumeration, never free text. The + ``TypeError`` otherwise lists the valid values.""" if isinstance(value, DdsFolder): return value.value - names = ", ".join(member.name for member in DdsFolder) hint = "" if isinstance(value, str): - match = next((m for m in DdsFolder if m.value == value.strip().strip("/").lower()), None) - hint = f" — write {match.name}, without quotes" if match else "" + match = closest_folder(value) + hint = f" (did you mean {match.name}, without quotes?)" if match else "" raise TypeError( - f"folder must be one of the DdsFolder values {names}, " - f"not {type(value).__name__} {value!r}{hint}" + f"folder must be a DdsFolder, not {type(value).__name__} {value!r}{hint}; " + f"{valid_folders_hint()}" ) @@ -84,6 +114,74 @@ def __repr__(self) -> str: return f"DdsFile(name={self.name!r}, data=<{len(self.data)} bytes>)" +@dataclass(frozen=True) +class DdsListing: + """The files :meth:`DdsController.list` found in one folder, sorted by name. + + ``patterns`` are the filters the names matched (empty: no filter). + Iterates, indexes and ``in``-tests like the tuple of names it holds. + """ + + target: str + names: tuple[str, ...] + patterns: tuple[str, ...] = () + + def __iter__(self) -> Iterator[str]: + return iter(self.names) + + def __len__(self) -> int: + return len(self.names) + + def __getitem__(self, index: int) -> str: + return self.names[index] + + def __contains__(self, name: object) -> bool: + return name in self.names + + def __repr__(self) -> str: + matching = f" matching {' or '.join(map(repr, self.patterns))}" if self.patterns else "" + if not self.names: + return f"no files{matching} in {self.target}" + lines = "".join(f"\n {name}" for name in self.names) + return f"{len(self.names)} file(s){matching} in {self.target}:{lines}" + + +def name_patterns(pattern: str | Iterable[str] | None) -> tuple[str, ...]: + """``pattern`` (see :meth:`DdsController.list`) as a tuple of non-empty patterns.""" + if pattern is None: + return () + items = [pattern] if isinstance(pattern, str) else list(pattern) + for item in items: + if not isinstance(item, str): + raise TypeError(f"pattern must be a str or a list of str, not {type(item).__name__}") + return tuple(dict.fromkeys(item.strip() for item in items if item.strip())) + + +def matches(name: str, patterns: tuple[str, ...]) -> bool: + """Whether ``name`` matches any of ``patterns``, ignoring case (no patterns: all do). + + - with ``*``, ``?`` or ``[...]`` it is a wildcard, matched against the + whole name and against the part after the last ``/``; + - starting with ``.`` it is an extension (``.pdf``, ``.tar.gz``); + - anything else is text the name contains (``report``). + """ + if not patterns: + return True + lowered = name.lower() + base = lowered.rsplit("/", 1)[-1] + for pattern in patterns: + wanted = pattern.lower() + if _GLOB_CHARS & set(wanted): + if fnmatch.fnmatchcase(lowered, wanted) or fnmatch.fnmatchcase(base, wanted): + return True + elif wanted.startswith("."): + if base.endswith(wanted): + return True + elif wanted in lowered: + return True + return False + + @dataclass(frozen=True) class PushReport: """What :meth:`DdsController.push` uploaded, in order.""" @@ -118,6 +216,14 @@ def __init__(self, existing: Iterable[str]) -> None: ) +class DdsFileNotFoundError(LookupError): + """:meth:`DdsController.get` asked for a file the folder does not have.""" + + def __init__(self, path: str, target: str) -> None: + self.path = path + super().__init__(f"no file {path.removeprefix(target + '/')!r} in {target}") + + class DdsController: """Uploads files to a catalog node's document folders in DDS. @@ -229,6 +335,53 @@ def push( uploaded.append(path) return PushReport(target, tuple(uploaded), overwrite) + def list( + self, + dds_path: str, + folder: DdsFolder, + pattern: str | Iterable[str] | None = None, + ) -> DdsListing: + """The names of the files in ``{dds_path}/{folder}``, sorted. + + ``dds_path`` and ``folder`` are checked as for :meth:`push`. A file in + a subfolder is named by its path below ``folder`` (``annex/a.pdf``). + A folder DDS has nothing in — or does not have yet — lists empty. + + ``pattern`` keeps only the names that match it, ignoring case; a list + of them keeps a name matching any (see :func:`matches`): + ``"*.pdf"`` / ``"report_202?.*"`` / ``"[ab]*"`` are wildcards, + ``".xlsx"`` an extension, ``"report"`` text in the name. + """ + patterns = name_patterns(pattern) + target = self.target(dds_path, folder) + prefix = f"{target}/" + names = set() + for path in self._documents_client().list(target): + cleaned = path.strip().strip("/") + name = cleaned[len(prefix) :] if cleaned.startswith(prefix) else cleaned + if name and name != target and matches(name, patterns): + names.add(name) + return DdsListing(target, tuple(sorted(names)), patterns) + + def get(self, dds_path: str, folder: DdsFolder, filename: str) -> DdsFile: + """The file ``filename`` of ``{dds_path}/{folder}``, with its bytes. + + ``dds_path`` and ``folder`` are checked as for :meth:`push`. + ``filename`` is a name :meth:`list` shows — a file in a subfolder is + ``annex/a.xlsx``. The :class:`DdsFile` is named after the file itself + (``a.xlsx``), so it can be pushed on as it is. A file DDS does not have + raises :class:`DdsFileNotFoundError`. + """ + target = self.target(dds_path, folder) + relative = require_relative_name(filename) + path = f"{target}/{relative}" + documents = self._documents_client() + data = documents.get(path) + if data is None: + raise DdsFileNotFoundError(path, target) + name = relative.rsplit("/", 1)[-1] + return DdsFile(name, data, mimetypes.guess_type(name)[0] or DEFAULT_CONTENT_TYPE) + # -- internals -------------------------------------------------------- def _documents_client(self) -> DocumentsClient: @@ -341,6 +494,18 @@ def _read_path(path: Path) -> DdsFile: return DdsFile(require_file_name(path.name), path.read_bytes()) +def require_relative_name(name: str) -> str: + """``name`` as a file name, or a path of file names below a folder + (``annex/a.xlsx``): every ``/``-separated part must be a valid file name + (see :func:`require_file_name`).""" + if not isinstance(name, str): + raise TypeError(f"file name must be a str, not {type(name).__name__}") + cleaned = name.strip().strip("/") + if not cleaned: + raise ValueError("file name must not be empty") + return "/".join(require_file_name(part) for part in cleaned.split("/")) + + def require_file_name(name: str) -> str: """``name`` as a valid single file name. diff --git a/src/eea_datalakehouse/dds_documents/session.py b/src/eea_datalakehouse/dds_documents/session.py index c5b0395..288ace4 100644 --- a/src/eea_datalakehouse/dds_documents/session.py +++ b/src/eea_datalakehouse/dds_documents/session.py @@ -3,13 +3,17 @@ dds = DdsSession(resolve_path=...) # %dds — writes files to DDS dds.push(METADATA, "bwd_2025.xml") dds.push(DELIVERABLES, "report.pdf", "annex/*.xlsx", overwrite=False) + dds.list(DELIVERABLES) # the file names already there + dds.list(DELIVERABLES, "*.pdf") # only some of them + dds.get(DELIVERABLES, "report.pdf") # saved into the notebook's folder - **The target path is the `%catalog` context.** `push(folder, *files)` puts - the files in `{context}/{folder}/` — the dataset folder `%catalog use(...)` - last set. `path=` picks another one, absolute or relative to the context, - exactly as `%catalog` resolves paths. The resolution is the ``resolve_path`` - callable the magic hands in; without one, `path=` is required and taken as - given. + the files in `{context}/{folder}/`, `list(folder)` shows the files there + and `get(folder, filename)` downloads one — the dataset folder + `%catalog use(...)` last set. `path=` picks + another one, absolute or relative to the context, exactly as `%catalog` + resolves paths. The resolution is the ``resolve_path`` callable the magic + hands in; without one, `path=` is required and taken as given. - **Every failure is a `DdsSessionError`**, so the magic can print it as one short line instead of a traceback (the original error is ``__cause__``). @@ -28,8 +32,10 @@ from __future__ import annotations +import builtins import os from collections.abc import Callable, Mapping +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -44,7 +50,17 @@ ) from ..session_utils import check_arg_types, find_dotenv, friendly_errors, read_dotenv_value from .client import DocumentsApiError, DocumentsClient -from .controller import DdsController, DdsFolder, DdsPushError, PushReport, folder_name +from .controller import ( + DdsController, + DdsFileNotFoundError, + DdsFolder, + DdsListing, + DdsPushError, + PushReport, + folder_name, + name_patterns, + require_relative_name, +) # What `%catalog` reads (see notebook/magics.py's _build_catalog_session). ENV_CATALOG_USER = "DREMIO_USERNAME" @@ -53,6 +69,7 @@ # Failures a notebook user can act on; anything else is a bug and keeps its traceback. _EXPECTED = ( DdsPushError, + DdsFileNotFoundError, DocumentsApiError, MissingCredentialsError, ValueError, @@ -62,6 +79,37 @@ ) +@dataclass(frozen=True) +class DdsDownload: + """A file `DdsSession.get` saved: where it came from in DDS and where it is now. + + It is the local file (``os.PathLike``), so ``open(download)``, + ``pd.read_csv(download)`` or ``%dds push(..., download)`` take it as is. + """ + + path: Path + source: str + size: int + content_type: str + + def __fspath__(self) -> str: + return os.fspath(self.path) + + def __repr__(self) -> str: + return f"got {self.source} ({_size_text(self.size)}) -> {self.path}" + + +def _size_text(size: int) -> str: + if size < 1024: + return f"{size} bytes" + value = size / 1024 + for unit in ("KB", "MB"): + if value < 1024: + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GB" + + class DdsSessionError(RuntimeError): """Any failure of a `DdsSession` call; the original error is ``__cause__``.""" @@ -83,7 +131,7 @@ def load_dds_creds(env: Mapping[str, str] | None = None) -> DremioCreds: class DdsSession: - """`push` / `target` / `folders` / `dds_base_url` for a notebook — what + """`push` / `list` / `get` / `target` / `folders` / `dds_base_url` for a notebook — what `%dds` dispatches onto. ``resolve_path(path)`` turns the ``path`` argument into the dataset's @@ -150,6 +198,79 @@ def push( dds_path, folder, flat, overwrite=overwrite, target_name=target_name ) + @_friendly + def list( + self, + folder: DdsFolder, + pattern: str | builtins.list[str] | tuple[str, ...] | None = None, + path: str | None = None, + ) -> DdsListing: + """The names of the files in `{path}/{folder}/` in DDS, sorted; `path` + defaults to the `%catalog` context. `folder` is compulsory. + + `pattern` keeps only the names that match it, ignoring case: a + wildcard (`"*.pdf"`, `"report_202?.*"`, `"[ab]*"`), an extension + (`".xlsx"`) or text in the name (`"report"`). A list keeps the names + matching any of them (`["*.pdf", "*.docx"]`).""" + folder_name(folder) + check_arg_types( + "list", + ("folder", folder, DdsFolder, False), + ("pattern", pattern, (str, builtins.list, tuple), True), + ("path", path, str, True), + ) + name_patterns(pattern) # a list of anything but str: no settings needed to say so + dds_path = self._dds_path(path) + DdsController.target(dds_path, folder) # bad folder or path: no settings needed + return self._dds().list(dds_path, folder, pattern) + + @_friendly + def get( + self, + folder: DdsFolder, + filename: str, + path: str | None = None, + output_dir: str | None = None, + overwrite: bool = True, + ) -> DdsDownload: + """Download `filename` from `{path}/{folder}/` in DDS into `output_dir` + (default: the notebook's folder) and return the local file; `path` + defaults to the `%catalog` context. `folder` and `filename` are + compulsory. A file DDS does not have at `{path}/{folder}/filename` is an + error. `overwrite=False` refuses to replace a local file of that + name (checked before downloading).""" + folder_name(folder) + check_arg_types( + "get", + ("folder", folder, DdsFolder, False), + ("filename", filename, str, False), + ("path", path, str, True), + ("output_dir", output_dir, str, True), + ("overwrite", overwrite, bool, False), + ) + dds_path = self._dds_path(path) + target = DdsController.target(dds_path, folder) # bad folder or path: no settings needed + relative = require_relative_name(filename) + local = (Path(output_dir).expanduser() if output_dir else Path.cwd()) / relative.rsplit( + "/", 1 + )[-1] + if local.is_dir(): + raise ValueError(f"{str(local)!r} is a folder; pass output_dir= to save elsewhere") + if local.exists() and not overwrite: + raise FileExistsError( + f"{str(local)!r} already exists; pass overwrite=True to replace it, " + "or output_dir= to save elsewhere" + ) + got = self._dds().get(dds_path, folder, relative) + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(got.data) + return DdsDownload( + local.resolve(), + f"{target}/{relative}", + len(got.data), + got.content_type or "application/octet-stream", + ) + @_friendly def target(self, folder: DdsFolder, path: str | None = None) -> str: """The DDS folder `push` would upload to, without uploading anything.""" @@ -157,8 +278,9 @@ def target(self, folder: DdsFolder, path: str | None = None) -> str: check_arg_types("target", ("folder", folder, DdsFolder, False), ("path", path, str, True)) return DdsController.target(self._dds_path(path), folder) - def folders(self) -> list[str]: - """The folders `push` accepts — the `DdsFolder` values, typed bare on a `%dds` line.""" + def folders(self) -> builtins.list[str]: + """The folders `push` and `list` accept — the `DdsFolder` values, typed + bare on a `%dds` line.""" return [member.name for member in DdsFolder] @_friendly diff --git a/src/eea_datalakehouse/notebook/magics.py b/src/eea_datalakehouse/notebook/magics.py index e4f87aa..27dd590 100644 --- a/src/eea_datalakehouse/notebook/magics.py +++ b/src/eea_datalakehouse/notebook/magics.py @@ -103,7 +103,7 @@ from ..catalog import Catalog from ..catalog.session import CatalogSession, CatalogSessionError -from ..dds_documents.controller import DdsFolder, PushReport +from ..dds_documents.controller import DdsFolder, PushReport, closest_folder, valid_folders_hint from ..dds_documents.session import DdsSession, DdsSessionError from ..dds_ingestion.session import IngestSession, IngestSessionError from ..geopackage.session import GpkgSession, GpkgSessionError @@ -126,7 +126,8 @@ ), "dds": ( '%dds push(DELIVERABLES, "report.pdf") | %dds push(METADATA, "a.xml", "b.xml")' - " (into the %catalog use(...) folder)" + ' | %dds list(DELIVERABLES) | %dds get(DELIVERABLES, "report.pdf")' + " (in the %catalog use(...) folder)" ), } @@ -348,8 +349,27 @@ "overwrite=True (default) replaces a file " "already there; overwrite=False uploads nothing if any is.", ), + ( + "list", + "Show the names of the files in path/folder/ in DDS, sorted. folder is compulsory, " + "as for push; path defaults to the %catalog use(...) context. pattern (optional) keeps " + 'only the names that match it, ignoring case: a wildcard ("*.pdf", "report_202?.*", ' + '"[ab]*"), an extension (".xlsx") or text in the name ("report"); a list keeps the ' + 'names matching any (["*.pdf", "*.docx"]). The result iterates ' + 'like a list of names, so `"report.pdf" in %dds list(DELIVERABLES)` works.', + ), + ( + "get", + "Download one file from path/folder/ in DDS and return the local file. folder and " + 'filename are compulsory (filename as list shows it, e.g. "annex/a.xlsx" for a ' + "subfolder); path defaults to the %catalog use(...) context. The file is saved in " + "output_dir (default: the notebook's folder) under its own name; overwrite=True " + "(default) replaces a local file of that name, overwrite=False refuses. A file DDS does " + "not have there is an error. The result is the local path, " + "so open(f), pd.read_csv(f) or %dds push(OTHERS, f) take it as is.", + ), ("target", "Show the DDS folder push would upload to, without uploading anything."), - ("folders", "List the folders push accepts."), + ("folders", "List the folders push, list and get accept."), ("dds_base_url", "Show the DDS URL push uses, and where it came from."), ] _DDS_HELP_NOTE = ( @@ -390,6 +410,25 @@ " check first, without uploading: %dds target(DELIVERABLES) or " '%dds target(DELIVERABLES, path="../wfd")\n' "\n" + "list examples — folder is compulsory, path= as for push:\n" + " %dds list(DELIVERABLES)\n" + ' %dds list(METADATA, path="../wfd")\n' + " names = %dds list(ASSESSMENTS)\n" + " pattern — optional, ignores case; a wildcard, an extension, text in the name, or a list:\n" + ' %dds list(DELIVERABLES, "*.pdf")\n' + ' %dds list(DELIVERABLES, ".xlsx")\n' + ' %dds list(DELIVERABLES, "report")\n' + ' %dds list(DELIVERABLES, "report_202?.*")\n' + ' %dds list(DELIVERABLES, ["*.pdf", "*.docx"], path="../wfd")\n' + "\n" + "get examples — folder and filename are compulsory, path= as for push:\n" + ' %dds get(DELIVERABLES, "report.pdf") # -> ./report.pdf\n' + ' f = %dds get(METADATA, "bwd.xml", path="../wfd", output_dir="downloads")\n' + ' %dds get(DELIVERABLES, "annex/a.xlsx", overwrite=False)\n' + "\n" + 'A folder that is not one of the eight (DELIVERABLE, "deliverables", left out) is an ' + "error that lists the valid ones.\n" + "\n" "Environment: DDS_BASE_URL — read from the first .env found in the notebook's folder or " "a parent when %dds starts, else from the kernel environment — and a Dremio identity: " "_DREMIO_USER/_DREMIO_PWD (as %ingest) or DREMIO_USERNAME/DREMIO_TOKEN (as %catalog)." @@ -403,6 +442,71 @@ } +# %dds methods whose first argument is the compulsory DdsFolder. +_DDS_FOLDER_METHODS = { + "push": 'push(DELIVERABLES, "report.pdf")', + "list": "list(DELIVERABLES)", + "get": 'get(DELIVERABLES, "report.pdf")', + "target": "target(DELIVERABLES)", +} + + +def _dds_folder_call(line: str) -> tuple[str, ast.expr | None] | None: + """``(method, folder argument)`` if ``line`` calls a %dds method that takes a + folder (the argument is ``None`` when it is left out), else ``None``.""" + text = line.strip() + if text.isidentifier(): + text = f"{text}()" + try: + call = ast.parse(text, mode="eval").body + except SyntaxError: + return None # _dispatch explains it + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id in _DDS_FOLDER_METHODS + ): + return None + folder = next((kw.value for kw in call.keywords if kw.arg == "folder"), None) + if folder is None and call.args and not isinstance(call.args[0], ast.Starred): + folder = call.args[0] + return call.func.id, folder + + +def _dds_missing_folder(line: str) -> str | None: + """The error for a %dds call that leaves out its compulsory folder, else ``None``.""" + found = _dds_folder_call(line) + if found is None or found[1] is not None: + return None + example = _DDS_FOLDER_METHODS[found[0]] + return f"{found[0]} needs a folder first, e.g. {example}; {valid_folders_hint()}" + + +def _dds_unknown_folder(line: str, exc: NameError | AttributeError) -> str | None: + """The error for a %dds folder that names no DdsFolder member — + ``push(DELIVERABLE, ...)`` or ``push(DdsFolder.REPORTS, ...)`` — else + ``None`` (the undefined name is something other than the folder).""" + found = _dds_folder_call(line) + if found is None or found[1] is None: + return None + node = found[1] + if isinstance(exc, NameError) and isinstance(node, ast.Name) and node.id == exc.name: + typed = node.id + elif ( + isinstance(exc, AttributeError) + and isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "DdsFolder" + and node.attr == exc.name + ): + typed = node.attr + else: + return None + match = closest_folder(typed) + hint = f" (did you mean {match.name}?)" if match else "" + return f"{typed} is not a DDS folder{hint}; {valid_folders_hint()}" + + _GPKG_HELP = [ ( "export", @@ -966,22 +1070,35 @@ def dds(self, line: str) -> Any: %dds push(DELIVERABLES, "report.pdf", "annex/*.xlsx") %dds push(METADATA, record, path="catalog.water.bwd", overwrite=False) + %dds list(DELIVERABLES) + %dds get(DELIVERABLES, "report.pdf") %dds target(ASSESSMENTS) """ if _is_help(line): _print_help_table(DdsSession, _DDS_HELP, "dds", note=_DDS_HELP_NOTE) return None + missing = _dds_missing_folder(line) + if missing is not None: + print(f"dds error: {missing}") + return None if self._dds_session is None: # Looked up at push time, so a `use` run after %dds started still counts. self._dds_session = _build_dds_session(self._context_target) - result = _dispatch( - self._dds_session, - line, - self.shell.user_ns, - "dds", - DdsSessionError, - extra_ns=_DDS_NAMESPACE, - ) + try: + result = _dispatch( + self._dds_session, + line, + self.shell.user_ns, + "dds", + DdsSessionError, + extra_ns=_DDS_NAMESPACE, + ) + except (NameError, AttributeError) as exc: + unknown = _dds_unknown_folder(line, exc) + if unknown is None: + raise # not the folder: an undefined notebook variable keeps its traceback + print(f"dds error: {unknown}") + return None if isinstance(result, PushReport): for dds_path in result.paths: _notify_dds_pushed(self.shell, dds_path) diff --git a/tests/dds_documents/conftest.py b/tests/dds_documents/conftest.py index a53d105..8521107 100644 --- a/tests/dds_documents/conftest.py +++ b/tests/dds_documents/conftest.py @@ -15,6 +15,11 @@ def file_url(path: str, base: str = DDS_URL) -> str: return f"{base}/api/v1/files/{path}" +def list_url(folder: str, base: str = DDS_URL) -> str: + """The folder's listing: its path with a trailing slash.""" + return f"{base}/api/v1/files/{folder}/" + + def not_found() -> httpx.Response: return httpx.Response(404, json={"error": "not_found", "message": "no such file"}) diff --git a/tests/dds_documents/test_client.py b/tests/dds_documents/test_client.py index 4bcb3bb..bf67663 100644 --- a/tests/dds_documents/test_client.py +++ b/tests/dds_documents/test_client.py @@ -21,9 +21,7 @@ def client() -> DocumentsClient: @respx.mock def test_put_sends_body_type_and_bearer(client: DocumentsClient) -> None: - route = respx.put(f"{BASE_URL}/api/v1/files/a/b/x.xml").mock( - return_value=httpx.Response(201) - ) + route = respx.put(f"{BASE_URL}/api/v1/files/a/b/x.xml").mock(return_value=httpx.Response(201)) client.put("/a/b/x.xml", b"", content_type="application/xml") @@ -49,9 +47,7 @@ def test_folders_reads_the_live_list(client: DocumentsClient) -> None: @respx.mock def test_put_encodes_spaces_in_path(client: DocumentsClient) -> None: - route = respx.put(f"{BASE_URL}/api/v1/files/a%20b/x.xml").mock( - return_value=httpx.Response(201) - ) + route = respx.put(f"{BASE_URL}/api/v1/files/a%20b/x.xml").mock(return_value=httpx.Response(201)) client.put("a b/x.xml", b"") @@ -84,24 +80,118 @@ def test_get_and_exists(client: DocumentsClient) -> None: return_value=httpx.Response(200, content=b"") ) respx.get(f"{BASE_URL}/api/v1/files/a/missing.xml").mock(return_value=httpx.Response(404)) + head = respx.head(f"{BASE_URL}/api/v1/files/a/x.xml").mock(return_value=httpx.Response(200)) + respx.head(f"{BASE_URL}/api/v1/files/a/missing.xml").mock(return_value=httpx.Response(404)) assert client.get("a/x.xml") == b"" - assert client.exists("a/x.xml") assert client.get("a/missing.xml") is None + assert client.exists("a/x.xml") assert not client.exists("a/missing.xml") + assert head.calls.last.request.headers["Authorization"] == "Bearer s3cr3t-pwd" + + +LIST_URL = f"{BASE_URL}/api/v1/files/a/b/" @respx.mock @pytest.mark.parametrize( - "payload", [{"files": [{"path": "a/x.xml"}, {"path": "a/y.xml"}]}, ["a/x.xml", "a/y.xml"]] + "payload", + [ + ["a/b/x.xml", "a/b/y.xml"], + {"files": [{"path": "a/b/x.xml"}, {"path": "a/b/y.xml"}]}, + {"items": [{"name": "x.xml", "type": "file"}, {"name": "y.xml", "type": "file"}]}, + {"entries": [{"key": "a/b/x.xml"}, {"key": "a/b/y.xml"}], "next_cursor": None}, + {"children": [{"rel_path": "x.xml"}, {"rel_path": "y.xml"}]}, + ], ) -def test_list(client: DocumentsClient, payload: object) -> None: - route = respx.get(f"{BASE_URL}/api/v1/files").mock( - return_value=httpx.Response(200, json=payload) +def test_list_reads_the_folder_listing(client: DocumentsClient, payload: object) -> None: + route = respx.get(LIST_URL).mock(return_value=httpx.Response(200, json=payload)) + + names = client.list("/a/b/") + + assert [name.rsplit("/", 1)[-1] for name in names] == ["x.xml", "y.xml"] + request = route.calls.last.request + assert request.url.path == "/api/v1/files/a/b/" # the trailing slash asks for a listing + assert request.url.params["limit"] == "200" + assert request.headers["Authorization"] == "Bearer s3cr3t-pwd" + + +@respx.mock +def test_list_leaves_subfolders_out(client: DocumentsClient) -> None: + respx.get(LIST_URL).mock( + return_value=httpx.Response( + 200, + json={ + "items": [ + {"name": "annex/", "type": "file"}, + {"name": "old", "type": "directory"}, + {"name": "older", "is_dir": True}, + {"name": "x.xml", "type": "file"}, + ] + }, + ) + ) + + assert client.list("a/b") == ["x.xml"] + + +@respx.mock +def test_list_follows_the_cursor_to_the_last_page(client: DocumentsClient) -> None: + route = respx.get(LIST_URL).mock( + side_effect=[ + httpx.Response(200, json={"items": ["x.xml"], "next_cursor": "p2"}), + httpx.Response(200, json={"items": ["y.xml"], "next_cursor": "p3"}), + httpx.Response(200, json={"items": ["z.xml"]}), + ] ) - assert client.list("/a/") == ["a/x.xml", "a/y.xml"] - assert route.calls.last.request.url.params["prefix"] == "a" + assert client.list("a/b") == ["x.xml", "y.xml", "z.xml"] + assert [call.request.url.params.get("cursor") for call in route.calls] == [None, "p2", "p3"] + + +@respx.mock +def test_list_stops_when_the_cursor_does_not_move(client: DocumentsClient) -> None: + route = respx.get(LIST_URL).mock( + return_value=httpx.Response(200, json={"items": ["x.xml"], "cursor": "same"}) + ) + + assert client.list("a/b") == ["x.xml", "x.xml"] + assert route.call_count == 2 + + +@respx.mock +def test_list_of_a_folder_dds_does_not_have_is_empty(client: DocumentsClient) -> None: + respx.get(LIST_URL).mock(return_value=httpx.Response(404)) + + assert client.list("a/b") == [] + + +@respx.mock +@pytest.mark.parametrize( + ("response", "message"), + [ + (httpx.Response(500, json={"message": "boom"}), "500 on GET /api/v1/files/a/b/: boom"), + ( + httpx.Response(200, json={"weird": 1}), + r"unrecognised folder listing \(top-level keys: weird\)", + ), + ( + httpx.Response(200, json={"items": [{"size": 3}]}), + r"entry without a path or name \(keys: size\)", + ), + ( + httpx.Response(200, text="", headers={"content-type": "text/html"}), + "expected a JSON folder listing, got 'text/html'", + ), + ], +) +def test_list_errors_say_what_came_back( + client: DocumentsClient, response: httpx.Response, message: str +) -> None: + respx.get(LIST_URL).mock(return_value=response) + + with pytest.raises(DocumentsApiError, match=message): + client.list("a/b") @pytest.mark.parametrize("bad", ["", "/", "a/../b", "a//b"]) diff --git a/tests/dds_documents/test_controller.py b/tests/dds_documents/test_controller.py index e086812..4487e8d 100644 --- a/tests/dds_documents/test_controller.py +++ b/tests/dds_documents/test_controller.py @@ -9,14 +9,26 @@ from eea_datalakehouse.dds_documents import ( DdsController, DdsFile, + DdsFileNotFoundError, DdsFilesExistError, DdsFolder, DdsPushError, DocumentsClient, ) -from eea_datalakehouse.dds_documents.controller import collect_files, folder_name +from eea_datalakehouse.dds_documents.controller import ( + closest_folder, + collect_files, + folder_name, + matches, + name_patterns, +) + +from .conftest import DATASET, DDS_URL, TARGET, file_url, list_url, not_found -from .conftest import DATASET, TARGET, file_url, not_found +VALID = ( + "valid folders are DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES, REFERENCES, " + "DELIVERABLES, OTHERS" +) @pytest.fixture @@ -56,18 +68,37 @@ def test_folder_name_is_the_members_value() -> None: @pytest.mark.parametrize( ("value", "message"), [ - ("metadata", "not str 'metadata' — write METADATA, without quotes"), - ("METADATA", "write METADATA, without quotes"), - ( - "maps", - r"must be one of the DdsFolder values DERIVED, METADATA, .*, OTHERS, not str 'maps'$", - ), + ("metadata", "not str 'metadata' \\(did you mean METADATA, without quotes\\?\\); "), + ("METADATA", "did you mean METADATA, without quotes"), + ("deliverable", "did you mean DELIVERABLES, without quotes"), + ("maps", "folder must be a DdsFolder, not str 'maps'; "), (None, "not NoneType None"), ], ) def test_folder_must_be_a_dds_folder(value: object, message: str) -> None: - with pytest.raises(TypeError, match=message): + with pytest.raises(TypeError, match=message) as raised: folder_name(value) # type: ignore[arg-type] + assert str(raised.value).endswith(VALID) + + +def test_an_unknown_folder_value_lists_the_valid_ones() -> None: + with pytest.raises(ValueError, match="'maps' is not a DDS folder") as raised: + DdsFolder("maps") + assert str(raised.value).endswith(VALID) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("deliverables", DdsFolder.DELIVERABLES), + ("Deliverable", DdsFolder.DELIVERABLES), + ("/metadata/", DdsFolder.METADATA), + ("analyses", DdsFolder.ANALYSES), + ("maps", None), + ], +) +def test_closest_folder(text: str, expected: DdsFolder | None) -> None: + assert closest_folder(text) is expected @pytest.mark.parametrize( @@ -173,10 +204,8 @@ def test_an_unknown_extension_goes_as_octet_stream(dds: DdsController) -> None: @respx.mock def test_no_overwrite_uploads_nothing_if_any_file_exists(dds: DdsController, files: Path) -> None: - respx.get(file_url(f"{TARGET}/report.pdf")).mock(return_value=not_found()) - respx.get(file_url(f"{TARGET}/notes.txt")).mock( - return_value=httpx.Response(200, content=b"old") - ) + respx.head(file_url(f"{TARGET}/report.pdf")).mock(return_value=httpx.Response(404)) + respx.head(file_url(f"{TARGET}/notes.txt")).mock(return_value=httpx.Response(200)) put = respx.put(url__startswith=file_url(TARGET)) with pytest.raises(DdsFilesExistError, match="notes.txt; pass overwrite=True") as info: @@ -193,7 +222,7 @@ def test_no_overwrite_uploads_nothing_if_any_file_exists(dds: DdsController, fil @respx.mock def test_no_overwrite_pushes_new_files_without_overwrite(dds: DdsController, files: Path) -> None: - respx.get(file_url(f"{TARGET}/report.pdf")).mock(return_value=not_found()) + respx.head(file_url(f"{TARGET}/report.pdf")).mock(return_value=httpx.Response(404)) put = respx.put(file_url(f"{TARGET}/report.pdf")).mock(return_value=httpx.Response(201)) dds.push(DATASET, DdsFolder.DELIVERABLES, files / "report.pdf", overwrite=False) @@ -204,7 +233,7 @@ def test_no_overwrite_pushes_new_files_without_overwrite(dds: DdsController, fil def test_a_string_folder_uploads_nothing(dds: DdsController, files: Path) -> None: with respx.mock(assert_all_called=False) as mock: put = mock.put(url__startswith=file_url("catalog")) - with pytest.raises(TypeError, match="write DELIVERABLES, without quotes"): + with pytest.raises(TypeError, match="did you mean DELIVERABLES"): dds.push(DATASET, "deliverables", files / "notes.txt") # type: ignore[arg-type] assert not put.called @@ -235,3 +264,163 @@ def test_bad_input_uploads_nothing(dds: DdsController, files: Path) -> None: def test_repr_hides_credentials(dds: DdsController) -> None: assert "s3cr3t" not in repr(dds) + + +# -- list --------------------------------------------------------------------- + + +@respx.mock +@pytest.mark.parametrize( + "paths", + [ + [f"{TARGET}/report.pdf", f"{TARGET}/annex/a.xlsx", f"/{TARGET}/notes.txt"], + ["report.pdf", "annex/a.xlsx", "notes.txt"], # names relative to the folder + ], +) +def test_list_names_the_files_in_the_folder(dds: DdsController, paths: list[str]) -> None: + route = respx.get(list_url(TARGET)).mock( + return_value=httpx.Response(200, json={"files": [{"path": p} for p in paths]}) + ) + + listing = dds.list(DATASET, DdsFolder.DELIVERABLES) + + assert route.calls.last.request.url.params["limit"] == "200" + assert listing.target == TARGET + assert listing.names == ("annex/a.xlsx", "notes.txt", "report.pdf") + assert list(listing) == list(listing.names) + assert len(listing) == 3 and listing[0] == "annex/a.xlsx" and "report.pdf" in listing + assert repr(listing) == (f"3 file(s) in {TARGET}:\n annex/a.xlsx\n notes.txt\n report.pdf") + + +@respx.mock +@pytest.mark.parametrize("response", [httpx.Response(200, json={"files": []}), not_found()]) +def test_an_empty_or_missing_folder_lists_empty( + dds: DdsController, response: httpx.Response +) -> None: + respx.get(list_url("catalog/water/bwd/metadata")).mock(return_value=response) + + listing = dds.list(DATASET, DdsFolder.METADATA) + + assert listing.names == () + assert repr(listing) == "no files in catalog/water/bwd/metadata" + + +NAMES = ["Report_2025.PDF", "report_2024.pdf", "annex/a.xlsx", "annex/b.csv", "notes.txt"] + + +@pytest.mark.parametrize( + ("pattern", "expected"), + [ + (None, sorted(NAMES)), + ("", sorted(NAMES)), + ("*.pdf", ["Report_2025.PDF", "report_2024.pdf"]), # case ignored + (".PDF", ["Report_2025.PDF", "report_2024.pdf"]), # an extension + ("pdf", ["Report_2025.PDF", "report_2024.pdf"]), # text in the name + ("report_202?.*", ["Report_2025.PDF", "report_2024.pdf"]), + ("*2025*", ["Report_2025.PDF"]), + ("[ab].*", ["annex/a.xlsx", "annex/b.csv"]), # the part after the last / + ("annex/*", ["annex/a.xlsx", "annex/b.csv"]), # or the whole name + ("annex", ["annex/a.xlsx", "annex/b.csv"]), + ([".xlsx", "*.csv"], ["annex/a.xlsx", "annex/b.csv"]), # any of a list + (("notes*",), ["notes.txt"]), + (".tx", []), # an extension matches whole: .txt is not .tx + ("*.docx", []), + ], +) +@respx.mock +def test_list_pattern_keeps_the_matching_names( + dds: DdsController, pattern: object, expected: list[str] +) -> None: + respx.get(list_url(TARGET)).mock(return_value=httpx.Response(200, json=NAMES)) + + listing = dds.list(DATASET, DdsFolder.DELIVERABLES, pattern) # type: ignore[arg-type] + + assert list(listing) == expected + + +@respx.mock +def test_a_filtered_listing_says_what_it_matched(dds: DdsController) -> None: + respx.get(list_url(TARGET)).mock(return_value=httpx.Response(200, json=NAMES)) + + pdfs = dds.list(DATASET, DdsFolder.DELIVERABLES, ["*.pdf", " *.pdf ", ".docx"]) + none = dds.list(DATASET, DdsFolder.DELIVERABLES, "*.zip") + + assert pdfs.patterns == ("*.pdf", ".docx") + assert repr(pdfs).startswith(f"2 file(s) matching '*.pdf' or '.docx' in {TARGET}:\n") + assert repr(none) == f"no files matching '*.zip' in {TARGET}" + + +def test_name_patterns_need_strings() -> None: + assert name_patterns(None) == () + assert name_patterns(" *.pdf ") == ("*.pdf",) + with pytest.raises(TypeError, match="pattern must be a str or a list of str, not int"): + name_patterns(["*.pdf", 3]) # type: ignore[list-item] + + +def test_matches_without_patterns_keeps_everything() -> None: + assert matches("anything", ()) + + +def test_list_needs_a_dds_folder_and_a_dataset(dds: DdsController) -> None: + with respx.mock(assert_all_called=False) as mock: + get = mock.get(url__startswith=DDS_URL) + with pytest.raises(TypeError, match="did you mean DELIVERABLES") as raised: + dds.list(DATASET, "deliverables") # type: ignore[arg-type] + assert str(raised.value).endswith(VALID) + with pytest.raises(ValueError, match="catalog root"): + dds.list("catalog", DdsFolder.DELIVERABLES) + + assert not get.called + + +# -- get ---------------------------------------------------------------------- + + +@respx.mock +def test_get_returns_the_file_named_after_itself(dds: DdsController) -> None: + route = respx.get(file_url(f"{TARGET}/annex/a.xlsx")).mock( + return_value=httpx.Response(200, content=b"PK-xlsx") + ) + + got = dds.get(DATASET, DdsFolder.DELIVERABLES, "/annex/a.xlsx") + + assert got == DdsFile( + "a.xlsx", b"PK-xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + assert route.calls.last.request.headers["Authorization"] == "Bearer s3cr3t-pwd" + + +def test_get_a_missing_file_is_an_error_without_listing_the_folder(dds: DdsController) -> None: + with respx.mock(assert_all_called=False) as mock: + mock.get(file_url(f"{TARGET}/annex/repot.pdf")).mock(return_value=not_found()) + listing = mock.get(url__startswith=f"{DDS_URL}/api/v1/files/{TARGET}/") + + with pytest.raises(DdsFileNotFoundError) as raised: + dds.get(DATASET, DdsFolder.DELIVERABLES, "annex/repot.pdf") + + assert str(raised.value) == f"no file 'annex/repot.pdf' in {TARGET}" + assert raised.value.path == f"{TARGET}/annex/repot.pdf" + assert not listing.called + + +@pytest.mark.parametrize( + ("folder", "filename", "error", "message"), + [ + ("deliverables", "a.pdf", TypeError, "did you mean DELIVERABLES"), + (DdsFolder.DELIVERABLES, "", ValueError, "must not be empty"), + (DdsFolder.DELIVERABLES, "../x.pdf", ValueError, "must not be a path"), + (DdsFolder.DELIVERABLES, "a?.pdf", ValueError, "not a valid file name"), + (DdsFolder.DELIVERABLES, 3, TypeError, "must be a str"), + ], +) +def test_get_bad_input_asks_dds_nothing( + dds: DdsController, folder: object, filename: object, error: type[Exception], message: str +) -> None: + with respx.mock(assert_all_called=False) as mock: + route = mock.get(url__startswith=DDS_URL) + with pytest.raises(error, match=message) as raised: + dds.get(DATASET, folder, filename) # type: ignore[arg-type] + if not isinstance(folder, DdsFolder): + assert str(raised.value).endswith(VALID) + + assert not route.called diff --git a/tests/dds_documents/test_session.py b/tests/dds_documents/test_session.py index 3fa48e4..e18daa3 100644 --- a/tests/dds_documents/test_session.py +++ b/tests/dds_documents/test_session.py @@ -18,7 +18,7 @@ from eea_datalakehouse.dds_ingestion.credentials import MissingCredentialsError from eea_datalakehouse.session_utils import find_dotenv, read_dotenv_value -from .conftest import DATASET, DDS_URL, TARGET, file_url +from .conftest import DATASET, DDS_URL, TARGET, file_url, list_url IDENTITY = {"_DREMIO_USER": "a", "_DREMIO_PWD": "pat-a"} ENV = {"DDS_BASE_URL": DDS_URL, **IDENTITY} @@ -75,10 +75,84 @@ def test_root_context_is_refused_before_any_settings() -> None: @pytest.mark.parametrize("folder", ["metadata", "converted", 42]) def test_a_folder_that_is_not_a_dds_folder_is_refused_before_any_settings(folder: object) -> None: - with pytest.raises(DdsSessionError, match="folder must be one of the DdsFolder values"): - _session(env={}).push(folder, REPORT) # type: ignore[arg-type] - with pytest.raises(DdsSessionError, match="folder must be one of the DdsFolder values"): - _session(env={}).target(folder) # type: ignore[arg-type] + calls = [ + lambda session: session.push(folder, REPORT), + lambda session: session.list(folder), + lambda session: session.get(folder, "report.pdf"), + lambda session: session.target(folder), + ] + for call in calls: + with pytest.raises(DdsSessionError, match="folder must be a DdsFolder") as raised: + call(_session(env={})) + assert "valid folders are DERIVED, METADATA," in str(raised.value) + assert str(raised.value).endswith("DELIVERABLES, OTHERS") + + +@respx.mock +def test_list_shows_the_files_in_the_context_folder() -> None: + route = respx.get(list_url(TARGET)).mock( + return_value=httpx.Response( + 200, json={"files": [{"path": f"{TARGET}/b.pdf"}, {"path": f"{TARGET}/a.pdf"}]} + ) + ) + + listing = _session().list(DdsFolder.DELIVERABLES) + + assert listing.names == ("a.pdf", "b.pdf") + request = route.calls.last.request + assert request.url.path == f"/api/v1/files/{TARGET}/" + assert request.headers["Authorization"] == "Bearer pat-a" + + +@respx.mock +def test_list_path_overrides_the_context() -> None: + route = respx.get(list_url("catalog/air/aq/metadata")).mock( + return_value=httpx.Response(200, json={"files": []}) + ) + + _session().list(DdsFolder.METADATA, path="catalog.air.aq") + + assert route.called + + +@respx.mock +def test_list_pattern_filters_the_names() -> None: + respx.get(list_url(TARGET)).mock( + return_value=httpx.Response(200, json=[f"{TARGET}/a.pdf", f"{TARGET}/b.docx"]) + ) + session = _session() + + assert list(session.list(DdsFolder.DELIVERABLES, "*.PDF")) == ["a.pdf"] + assert list(session.list(DdsFolder.DELIVERABLES, [".pdf", ".docx"])) == ["a.pdf", "b.docx"] + assert list(session.list(DdsFolder.DELIVERABLES, pattern="b")) == ["b.docx"] + + +@pytest.mark.parametrize( + ("pattern", "message"), + [(3, r"\(pattern\) must be str or list or tuple or None"), (["*.pdf", 3], "list of str")], +) +def test_a_bad_pattern_needs_no_settings(pattern: object, message: str) -> None: + with pytest.raises(DdsSessionError, match=message): + _session(env={}).list(DdsFolder.DELIVERABLES, pattern) # type: ignore[arg-type] + + +def test_list_without_a_context_or_with_a_bad_path_is_a_session_error() -> None: + with pytest.raises(DdsSessionError, match="no DDS target path"): + _session(None, env={}).list(DdsFolder.METADATA) + with pytest.raises(DdsSessionError, match="catalog root"): + _session("catalog", env={}).list(DdsFolder.METADATA) + with pytest.raises(DdsSessionError, match=r"\(path\) must be str"): + _session(env={}).list(DdsFolder.METADATA, path=3) # type: ignore[arg-type] + + +@respx.mock +def test_list_server_error_is_a_session_error() -> None: + respx.get(list_url("catalog/water/bwd/metadata")).mock( + return_value=httpx.Response(500, text="down") + ) + + with pytest.raises(DdsSessionError, match="500"): + _session().list(DdsFolder.METADATA) def test_a_missing_file_is_a_session_error(tmp_path: Path) -> None: @@ -321,3 +395,86 @@ def test_read_dotenv_value_last_assignment_wins(tmp_path: Path) -> None: assert read_dotenv_value(dotenv, "DDS_BASE_URL") == "two" assert read_dotenv_value(dotenv, "MISSING") is None + + +# -- get ---------------------------------------------------------------------- + + +@respx.mock +def test_get_saves_the_file_in_the_working_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + respx.get(file_url(f"{TARGET}/report.pdf")).mock( + return_value=httpx.Response(200, content=b"%PDF-1") + ) + + got = _session().get(DdsFolder.DELIVERABLES, "report.pdf") + + assert got.path == tmp_path / "report.pdf" + assert (tmp_path / "report.pdf").read_bytes() == b"%PDF-1" + assert Path(got).read_bytes() == b"%PDF-1" # os.PathLike: the local file + assert got.source == f"{TARGET}/report.pdf" + assert (got.size, got.content_type) == (6, "application/pdf") + assert repr(got) == f"got {TARGET}/report.pdf (6 bytes) -> {tmp_path / 'report.pdf'}" + + +@respx.mock +def test_get_path_and_output_dir(tmp_path: Path) -> None: + respx.get(file_url("catalog/air/aq/metadata/annex/aq.xml")).mock( + return_value=httpx.Response(200, content=b"" * 600) + ) + + got = _session().get( + DdsFolder.METADATA, "annex/aq.xml", path="catalog.air.aq", output_dir=str(tmp_path / "dl") + ) + + assert got.path == tmp_path / "dl" / "aq.xml" + assert got.path.read_bytes() == b"" * 600 + assert "(2.3 KB)" in repr(got) + + +@respx.mock +def test_get_overwrite(tmp_path: Path) -> None: + respx.get(file_url(f"{TARGET}/report.pdf")).mock( + return_value=httpx.Response(200, content=b"new") + ) + (tmp_path / "report.pdf").write_bytes(b"old") + session = _session() + + with pytest.raises(DdsSessionError, match="already exists; pass overwrite=True"): + session.get(DdsFolder.DELIVERABLES, "report.pdf", output_dir=str(tmp_path), overwrite=False) + assert (tmp_path / "report.pdf").read_bytes() == b"old" + + session.get(DdsFolder.DELIVERABLES, "report.pdf", output_dir=str(tmp_path)) + assert (tmp_path / "report.pdf").read_bytes() == b"new" + + +@respx.mock +def test_get_a_missing_file_is_a_session_error(tmp_path: Path) -> None: + respx.get(file_url(f"{TARGET}/repot.pdf")).mock(return_value=httpx.Response(404)) + + with pytest.raises(DdsSessionError, match=f"^no file 'repot.pdf' in {TARGET}$"): + _session().get(DdsFolder.DELIVERABLES, "repot.pdf", output_dir=str(tmp_path)) + assert not (tmp_path / "repot.pdf").exists() + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"filename": 3}, r"\(filename\) must be str"), + ({"filename": "a/../b.pdf"}, "must not be a path"), + ({"path": "catalog"}, "catalog root"), + ({"overwrite": "yes"}, r"\(overwrite\) must be bool"), + ], +) +def test_get_bad_arguments_need_no_settings(kwargs: dict[str, object], message: str) -> None: + kwargs.setdefault("filename", "report.pdf") + with pytest.raises(DdsSessionError, match=message): + _session(env={}).get(DdsFolder.DELIVERABLES, **kwargs) # type: ignore[arg-type] + + +def test_get_into_a_folder_of_that_name_is_refused(tmp_path: Path) -> None: + (tmp_path / "report.pdf").mkdir() + with pytest.raises(DdsSessionError, match="is a folder"): + _session(env={}).get(DdsFolder.DELIVERABLES, "report.pdf", output_dir=str(tmp_path)) diff --git a/tests/notebook/test_magics.py b/tests/notebook/test_magics.py index 0617a33..c164d7f 100644 --- a/tests/notebook/test_magics.py +++ b/tests/notebook/test_magics.py @@ -12,6 +12,7 @@ import pytest from eea_datalakehouse.catalog.session import CatalogCommitError +from eea_datalakehouse.dds_documents import DdsSession from eea_datalakehouse.dds_ingestion.session import IngestSession from eea_datalakehouse.notebook import magics as magics_module from eea_datalakehouse.notebook.magics import EEALakehouseMagics @@ -467,6 +468,20 @@ def push( f"{target}/x", (f"{target}/x/a.pdf", f"{target}/x/b.pdf"), overwrite ) + def list(self, folder: object, pattern: object = None, path: str | None = None) -> object: + target = self._resolve_path(path) + self.calls.append((folder, (), target, True)) + matching = f" matching {pattern}" if pattern else "" + return f"listed {folder}{matching} in {target}" + + def get(self, folder: object, filename: str, path: str | None = None) -> object: + target = self._resolve_path(path) + self.calls.append((folder, (filename,), target, True)) + return f"got {filename} from {target}/{folder}" + + def target(self, folder: object, path: str | None = None) -> object: + return f"{self._resolve_path(path)}/{folder}" + def test_dds_pushes_to_the_catalog_context_with_bare_folder_names( ip: Any, monkeypatch: pytest.MonkeyPatch @@ -511,6 +526,113 @@ def test_dds_folder_names_win_over_notebook_variables( ] +def test_dds_list_shows_the_catalog_context_folder( + ip: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(magics_module, "_build_dds_session", _FakeDdsSession) + _magics_instance(ip)._pending_context = "catalog.water.bwd" + + assert ip.run_line_magic("dds", "list(DELIVERABLES)") == ( + "listed deliverables in catalog.water.bwd" + ) + assert ip.run_line_magic("dds", 'list(METADATA, path="catalog.air")') == ( + "listed metadata in catalog.air" + ) + assert ip.run_line_magic("dds", 'list(DELIVERABLES, ["*.pdf", ".docx"])') == ( + "listed deliverables matching ['*.pdf', '.docx'] in catalog.water.bwd" + ) + + +_VALID_FOLDERS = ( + "valid folders are DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES, REFERENCES, " + "DELIVERABLES, OTHERS" +) + + +@pytest.mark.parametrize( + ("line", "message"), + [ + ( + 'push(DELIVERABLE, "a.pdf")', + "DELIVERABLE is not a DDS folder (did you mean DELIVERABLES?); ", + ), + ("list(deliverables)", "deliverables is not a DDS folder (did you mean DELIVERABLES?); "), + ("target(REPORTS)", "REPORTS is not a DDS folder; "), + ("list(DdsFolder.REPORTS)", "REPORTS is not a DDS folder; "), + ( + 'push(folder=Metadata, files="a")', + "Metadata is not a DDS folder (did you mean METADATA?); ", + ), + ("list()", "list needs a folder first, e.g. list(DELIVERABLES); "), + ("get()", 'get needs a folder first, e.g. get(DELIVERABLES, "report.pdf"); '), + ( + 'get(Deliverables, "a.pdf")', + "Deliverables is not a DDS folder (did you mean DELIVERABLES?); ", + ), + ("list", "list needs a folder first"), + ('push(path="catalog.x")', "push needs a folder first"), + ], +) +def test_dds_a_wrong_or_missing_folder_lists_the_valid_ones( + ip: Any, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + line: str, + message: str, +) -> None: + monkeypatch.setattr(magics_module, "_build_dds_session", _FakeDdsSession) + _magics_instance(ip)._pending_context = "catalog.water.bwd" + + assert ip.run_line_magic("dds", line) is None + + out = capsys.readouterr().out.strip() + assert out.startswith(f"dds error: {message}") + assert out.endswith(_VALID_FOLDERS) + session = _magics_instance(ip)._dds_session + assert session is None or session.calls == [] + + +def test_dds_get_downloads_from_the_catalog_context_folder( + ip: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(magics_module, "_build_dds_session", _FakeDdsSession) + _magics_instance(ip)._pending_context = "catalog.water.bwd" + + assert ip.run_line_magic("dds", 'get(DELIVERABLES, "report.pdf")') == ( + "got report.pdf from catalog.water.bwd/deliverables" + ) + assert ip.run_line_magic("dds", 'get(METADATA, "a.xml", path="catalog.air")') == ( + "got a.xml from catalog.air/metadata" + ) + + +def test_dds_get_without_a_filename_shows_the_usage( + ip: Any, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr( + magics_module, + "_build_dds_session", + lambda resolve: DdsSession(resolve_path=resolve, env={}, dotenv_path="/nonexistent/.env"), + ) + _magics_instance(ip)._pending_context = "catalog.water.bwd" + + assert ip.run_line_magic("dds", "get(DELIVERABLES)") is None + + out = capsys.readouterr().out + assert "dds error:" in out and "filename" in out + assert "get(folder, filename, path=None, output_dir=None, overwrite=True)" in out + + +def test_dds_an_undefined_file_variable_keeps_its_name_error( + ip: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(magics_module, "_build_dds_session", _FakeDdsSession) + _magics_instance(ip)._pending_context = "catalog.water.bwd" + + with pytest.raises(NameError, match="no_such_record"): + ip.run_line_magic("dds", "push(METADATA, no_such_record)") + + def test_context_target_resolves_path_like_catalog( ip: Any, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -577,8 +699,12 @@ def test_dds_help_lists_methods_without_building_a_session( assert any(option in line or option.strip('"') in line for line in examples), option assert all(line.count("(") == line.count(")") for line in examples) # each fits on one line table_html = displayed[0].data - for name in ("push", "target", "folders", "dds_base_url"): + for name in ("push", "list", "get", "target", "folders", "dds_base_url"): assert f">{name}<" in table_html + assert "%dds list(DELIVERABLES)" in out + assert '%dds get(DELIVERABLES, "report.pdf")' in out + for example in ('"*.pdf"', '".xlsx"', '"report"', '["*.pdf", "*.docx"]'): + assert f"%dds list(DELIVERABLES, {example}" in out assert "folder, files..., path=None, overwrite=True, target_name=''" in table_html assert "DERIVED, METADATA, METHODOLOGY, ASSESSMENTS, ANALYSES" in table_html assert _magics_instance(ip)._dds_session is None