Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
recursive-include cfa/dataops/reporting/reports
recursive-include cfa/dataops/etl/transform_templates
include cfa/dataops/config.ini
include cfa/dataops/py.typed
41 changes: 35 additions & 6 deletions cfa/dataops/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from io import BytesIO
from pathlib import PurePosixPath
from types import SimpleNamespace
from typing import Any, Literal, overload
from typing import TYPE_CHECKING, Any, Literal, overload

import pandas as pd
import polars as pl
Expand Down Expand Up @@ -83,11 +83,40 @@ def get_all_catalogs() -> list:
report_namespaces = get_dataset_dot_path(all_reports_ns_map)


class CatalogNamespace(SimpleNamespace):
"""Recursive namespace wrapper for statically-typed catalog access.

Runtime instances still behave like ``SimpleNamespace`` objects, but the
declared attributes let editors resolve endpoint methods on dynamic access
chains such as ``datacat.public.team.dataset.load``.
"""

if TYPE_CHECKING:
load: "BlobEndpoint"
extract: "BlobEndpoint"
data: "BlobEndpoint"
_ledger_endpoint: "BlobEndpoint"
config: dict[str, Any]
__namespace_list__: list[str]

def __getattr__(self, name: str) -> "CatalogNamespace":
raise AttributeError(
f"{type(self).__name__!s} object has no attribute {name!r}"
)


class DatasetEndpoint:
"""The DatasetEndpoint class for including in the datacat namespace.
This ends the namespace branching at a config file and creates all the
blob endpoints for each 'stage' of the config (e.g., extract, load, stage_01)."""

if TYPE_CHECKING:
config: dict[str, Any]
load: "BlobEndpoint"
extract: "BlobEndpoint"
data: "BlobEndpoint"
_ledger_endpoint: "BlobEndpoint"

def __init__(self, config_path: str, defaults: dict, ns: str):
"""Basic functionality to interact with datasets to be included
via the datasets configs.
Expand Down Expand Up @@ -685,7 +714,7 @@ def save_dir_to_blob(
)


def dict_to_sn(d: Any, defaults: dict = None, ns: str = "") -> SimpleNamespace:
def dict_to_sn(d: Any, defaults: dict | None = None, ns: str = "") -> CatalogNamespace:
"""Simple recursive namespace construction

Args:
Expand All @@ -694,9 +723,9 @@ def dict_to_sn(d: Any, defaults: dict = None, ns: str = "") -> SimpleNamespace:
ns (str, optional): the current namespace path. Defaults to ''.

Returns:
SimpleNamespace: namespace representation
CatalogNamespace: namespace representation
"""
x = SimpleNamespace()
x = CatalogNamespace()
ns_prefix = f"{ns}." if ns != "" else ""
_ = [
setattr(
Expand Down Expand Up @@ -737,9 +766,9 @@ def dict_to_sn(d: Any, defaults: dict = None, ns: str = "") -> SimpleNamespace:
rc.append(report_dict_to_sn({k: all_reports_ns_map[k]}))
combined_reports_dict = {key: value for ns in rc for key, value in vars(ns).items()}

datacat = SimpleNamespace(**combined_dict)
datacat: CatalogNamespace = CatalogNamespace(**combined_dict)
datacat.__setattr__("__namespace_list__", dataset_namespaces)
reportcat = SimpleNamespace(**combined_reports_dict)
reportcat: CatalogNamespace = CatalogNamespace(**combined_reports_dict)
reportcat.__setattr__("__namespace_list__", report_namespaces)


Expand Down
Empty file added cfa/dataops/py.typed
Empty file.
3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ dependencies = [

[tool.poetry]
packages = [{ include = "cfa" }]
include = [
{ path = "cfa/dataops/py.typed", format = ["sdist", "wheel"] },
]


[tool.poetry.group.dev.dependencies]
Expand Down
Loading