From b900532b7bcd766cfb4f753e515756a3081a9d39 Mon Sep 17 00:00:00 2001 From: mb-jp Date: Mon, 16 Oct 2023 11:55:14 +0200 Subject: [PATCH 01/12] init b --- .../web/_responseAsFileObject.py | 14 ++ macrobond_data_api/web/_web_api_revision.py | 3 +- macrobond_data_api/web/_web_only_api.py | 9 +- macrobond_data_api/web/web_api.py | 2 + .../web_types/data_package_list_context.py | 140 ++++++++++++++++++ scripts/lint_tools.py | 4 +- 6 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 macrobond_data_api/web/_responseAsFileObject.py create mode 100644 macrobond_data_api/web/web_types/data_package_list_context.py diff --git a/macrobond_data_api/web/_responseAsFileObject.py b/macrobond_data_api/web/_responseAsFileObject.py new file mode 100644 index 00000000..18ef73e1 --- /dev/null +++ b/macrobond_data_api/web/_responseAsFileObject.py @@ -0,0 +1,14 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from requests import Response + + +class _ResponseAsFileObject: + def __init__(self, response: "Response", chunk_size: int = 65536) -> None: + self.data = response.iter_content(chunk_size=chunk_size) + + def read(self, n: int) -> bytes: + if n == 0: + return b"" + return next(self.data, b"") diff --git a/macrobond_data_api/web/_web_api_revision.py b/macrobond_data_api/web/_web_api_revision.py index 71db3635..0226ad4e 100644 --- a/macrobond_data_api/web/_web_api_revision.py +++ b/macrobond_data_api/web/_web_api_revision.py @@ -22,6 +22,7 @@ from .session import ProblemDetailsException, Session, _raise_on_error +from ._responseAsFileObject import _ResponseAsFileObject if TYPE_CHECKING: # pragma: no cover from .web_api import WebApi @@ -276,7 +277,7 @@ def get_many_series_with_revisions( _create_web_revision_h_request(requests_chunkd), stream=True ) as response: _raise_on_error(response) - ijson_items = ijson.items(response.raw, "item") + ijson_items = ijson.items(_ResponseAsFileObject(response), "item") item: "SeriesWithVintagesResponse" for item in ijson_items: error_code = item.get("errorCode") diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index 5e7c970d..cfcae6c1 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -5,6 +5,7 @@ from macrobond_data_api.common.types import SearchResultLong from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 +from .web_types.data_package_list_context import DataPackageListContext, _DataPackageListContext from .web_types.data_package_list_state import DataPackageListState from .web_types.data_pacakge_list_item import DataPackageListItem @@ -14,6 +15,8 @@ from .session import _raise_on_error from .subscription_list import SubscriptionList +from ._responseAsFileObject import _ResponseAsFileObject + if TYPE_CHECKING: # pragma: no cover from macrobond_data_api.common.types import SearchFilter @@ -161,7 +164,7 @@ def get_data_package_list_iterative( with self._session.get("v1/series/getdatapackagelist", params=params, stream=True) as response: _raise_on_error(response) - ijson_parse = ijson.parse(response.raw) + ijson_parse = ijson.parse(_ResponseAsFileObject(response)) ( time_stamp_for_if_modified_since, @@ -186,6 +189,10 @@ def get_data_package_list_iterative( return body +def _get_data_package_list_iterative_2(self: "WebApi", if_modified_since: datetime = None) -> DataPackageListContext: + return _DataPackageListContext(if_modified_since, self) + + # Search diff --git a/macrobond_data_api/web/web_api.py b/macrobond_data_api/web/web_api.py index 2c404907..e7a60b16 100644 --- a/macrobond_data_api/web/web_api.py +++ b/macrobond_data_api/web/web_api.py @@ -4,6 +4,7 @@ entity_search_multi_filter_long, get_data_package_list, get_data_package_list_iterative, + _get_data_package_list_iterative_2, subscription_list, ) from ._web_api_metadata import metadata_list_values, metadata_get_attribute_information, metadata_get_value_information @@ -71,6 +72,7 @@ def session(self) -> Session: get_data_package_list = get_data_package_list get_data_package_list_iterative = get_data_package_list_iterative + _get_data_package_list_iterative_2 = _get_data_package_list_iterative_2 entity_search_multi_filter_long = entity_search_multi_filter_long subscription_list = subscription_list diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py new file mode 100644 index 00000000..46843a0d --- /dev/null +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -0,0 +1,140 @@ +from abc import ABC, abstractmethod +from datetime import datetime +from typing import TYPE_CHECKING, Any, Optional, Tuple, cast, Iterable, Iterator + +import ijson + +from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 + +from ..web_types.data_package_list_state import DataPackageListState +from ..web_types.data_package_body import DataPackageBody +from ..session import _raise_on_error +from .._responseAsFileObject import _ResponseAsFileObject + +if TYPE_CHECKING: # pragma: no cover + from ..web_api import WebApi + from requests import Response + +# work in progress + + +class DataPackageListContext(ABC): + @abstractmethod + def __enter__(self) -> "DataPackageListIterable": + ... + + @abstractmethod + def __exit__(self, exception_type: Any, exception_value: Any, traceback: Any) -> None: + ... + + +class DataPackageListIterable(ABC, Iterable[Tuple[str, datetime]]): + @property + @abstractmethod + def body(self) -> DataPackageBody: + """_""" + + +class _DataPackageListContext(DataPackageListContext, DataPackageListIterable, Iterator[Tuple[str, datetime]]): + response_: Optional["Response"] + _ijson_parse: Any + _body: Optional[DataPackageBody] + + @property + def body(self) -> DataPackageBody: + return cast(DataPackageBody, self._body) + + def __init__(self, if_modified_since: Optional[datetime], webApi: "WebApi") -> None: + self._if_modified_since = if_modified_since + self._webApi: Optional["WebApi"] = webApi + self._iterator_started = False + + def __enter__(self) -> "_DataPackageListContext": + params = {} + if self._if_modified_since: + params["ifModifiedSince"] = self._if_modified_since.isoformat() + + if self._webApi is None: + raise Exception("obj is closed") + + try: + self.response_ = self._webApi._session.get("v1/series/getdatapackagelist", params=params, stream=True) + self._webApi = None + + _raise_on_error(self.response_) + self._ijson_parse = ijson.parse(_ResponseAsFileObject(self.response_)) + self._set_body() + + return self + + except Exception as e: + self.__exit__(None, None, None) + raise e + + def __exit__(self, exception_type: Any, exception_value: Any, traceback: Any) -> None: + self._webApi = None + if self.response_: + self.response_.close() + self.response_ = None + + def _set_body(self) -> None: + time_stamp_for_if_modified_since: Optional[datetime] = None + download_full_list_on_or_after: Optional[datetime] = None + state: Optional[DataPackageListState] = None + for prefix, event, value in self._ijson_parse: + if prefix == "timeStampForIfModifiedSince": + if event != "string": + raise Exception("bad format: timeStampForIfModifiedSince is not a string") + time_stamp_for_if_modified_since = _parse_iso8601(value) + elif prefix == "downloadFullListOnOrAfter": + if event != "string": + raise Exception("bad format: downloadFullListOnOrAfter is not a string") + download_full_list_on_or_after = _parse_iso8601(value) + elif prefix == "state": + if event != "number": + raise Exception("bad format: state is not a number") + state = DataPackageListState(value) + elif event == "start_array": + if prefix != "entities": + raise Exception("bad format: event start_array do not have a prefix of entities") + break + + if state is None: + raise Exception("bad format: state was not found") + if time_stamp_for_if_modified_since is None: + raise Exception("bad format: timeStampForIfModifiedSince was not found") + if not self._if_modified_since and download_full_list_on_or_after is None: + raise Exception("bad format: downloadFullListOnOrAfter was not found") + + self._body = DataPackageBody(time_stamp_for_if_modified_since, download_full_list_on_or_after, state) + + def __iter__(self) -> "_DataPackageListContext": + if self._iterator_started: + raise Exception("iterator has already started") + self._iterator_started = True + return self + + def __next__(self) -> Tuple[str, datetime]: + name = "" + modified: Optional[datetime] = None + + while True: + prefix, event, value = next(self._ijson_parse) + if event == "end_map": + if name == "": + raise Exception("bad format: name was not found") + if modified is None: + raise Exception("bad format: modified was not found") + return (name, modified) + + if event == "end_array": + raise StopIteration() + + if prefix == "entities.item.name": + if event != "string": + raise Exception("bad format: entities.item.name is not a string") + name = value + elif prefix == "entities.item.modified": + if event != "string": + raise Exception("bad format: entities.item.modified is not a string") + modified = _parse_iso8601(value) diff --git a/scripts/lint_tools.py b/scripts/lint_tools.py index 74251dc9..011dc064 100644 --- a/scripts/lint_tools.py +++ b/scripts/lint_tools.py @@ -9,7 +9,7 @@ class Mypy(WorkItem): # TODO: @mb-jp use --strict for mypy async def run(self) -> None: - await self.python_run("mypy", ". --show-error-codes --exclude .env --python-version 3.8") + await self.python_run("mypy", ". --show-error-codes --exclude .env --exclude test.py --python-version 3.8") class Pylint(WorkItem): @@ -19,7 +19,7 @@ async def run(self) -> None: class PyCodeStyle(WorkItem): async def run(self) -> None: - await self.python_run("pycodestyle", "--count . --exclude=.env") + await self.python_run("pycodestyle", "--count . --exclude=.env,test.py") class Black(WorkItem): From 7aa15f77a9b6e61972aa1d58a279fd58df05903c Mon Sep 17 00:00:00 2001 From: mb-jp Date: Mon, 16 Oct 2023 16:03:50 +0200 Subject: [PATCH 02/12] wip --- .../web/_responseAsFileObject.py | 14 -- macrobond_data_api/web/_web_api_revision.py | 4 +- macrobond_data_api/web/_web_only_api.py | 13 +- macrobond_data_api/web/session.py | 10 + .../web_types/data_package_list_context.py | 229 +++++++++++------- 5 files changed, 153 insertions(+), 117 deletions(-) delete mode 100644 macrobond_data_api/web/_responseAsFileObject.py diff --git a/macrobond_data_api/web/_responseAsFileObject.py b/macrobond_data_api/web/_responseAsFileObject.py deleted file mode 100644 index 18ef73e1..00000000 --- a/macrobond_data_api/web/_responseAsFileObject.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: # pragma: no cover - from requests import Response - - -class _ResponseAsFileObject: - def __init__(self, response: "Response", chunk_size: int = 65536) -> None: - self.data = response.iter_content(chunk_size=chunk_size) - - def read(self, n: int) -> bytes: - if n == 0: - return b"" - return next(self.data, b"") diff --git a/macrobond_data_api/web/_web_api_revision.py b/macrobond_data_api/web/_web_api_revision.py index 0226ad4e..8b43e97f 100644 --- a/macrobond_data_api/web/_web_api_revision.py +++ b/macrobond_data_api/web/_web_api_revision.py @@ -20,9 +20,7 @@ from macrobond_data_api.common.types._repr_html_sequence import _ReprHtmlSequence from ._split_in_to_chunks import split_in_to_chunks -from .session import ProblemDetailsException, Session, _raise_on_error - -from ._responseAsFileObject import _ResponseAsFileObject +from .session import ProblemDetailsException, Session, _raise_on_error, _ResponseAsFileObject if TYPE_CHECKING: # pragma: no cover from .web_api import WebApi diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index cfcae6c1..808d4508 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -5,18 +5,16 @@ from macrobond_data_api.common.types import SearchResultLong from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 -from .web_types.data_package_list_context import DataPackageListContext, _DataPackageListContext +from .web_types.data_package_list_context import DataPackageListContextManager from .web_types.data_package_list_state import DataPackageListState from .web_types.data_pacakge_list_item import DataPackageListItem from .web_types.data_package_list import DataPackageList from .web_types.data_package_body import DataPackageBody -from .session import _raise_on_error +from .session import _raise_on_error, _ResponseAsFileObject from .subscription_list import SubscriptionList -from ._responseAsFileObject import _ResponseAsFileObject - if TYPE_CHECKING: # pragma: no cover from macrobond_data_api.common.types import SearchFilter @@ -189,8 +187,11 @@ def get_data_package_list_iterative( return body -def _get_data_package_list_iterative_2(self: "WebApi", if_modified_since: datetime = None) -> DataPackageListContext: - return _DataPackageListContext(if_modified_since, self) +# i need a good name ! +def _get_data_package_list_iterative_2( + self: "WebApi", if_modified_since: datetime = None +) -> DataPackageListContextManager: + return DataPackageListContextManager(if_modified_since, self) # Search diff --git a/macrobond_data_api/web/session.py b/macrobond_data_api/web/session.py index 5defaef9..5d9fc622 100644 --- a/macrobond_data_api/web/session.py +++ b/macrobond_data_api/web/session.py @@ -54,6 +54,16 @@ def _raise_on_error(response: "Response", non_error_status: Sequence[int] = None raise HttpException(response) +class _ResponseAsFileObject: + def __init__(self, response: "Response", chunk_size: int = 65536) -> None: + self.data = response.iter_content(chunk_size=chunk_size) + + def read(self, n: int) -> bytes: + if n == 0: + return b"" + return next(self.data, b"") + + class Session: @property def metadata(self) -> MetadataMethods: diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py index 46843a0d..9396628f 100644 --- a/macrobond_data_api/web/web_types/data_package_list_context.py +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -1,15 +1,12 @@ -from abc import ABC, abstractmethod from datetime import datetime -from typing import TYPE_CHECKING, Any, Optional, Tuple, cast, Iterable, Iterator +from typing import TYPE_CHECKING, Any, Optional, Tuple, Iterable, Iterator import ijson from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 from ..web_types.data_package_list_state import DataPackageListState -from ..web_types.data_package_body import DataPackageBody -from ..session import _raise_on_error -from .._responseAsFileObject import _ResponseAsFileObject +from ..session import _raise_on_error, _ResponseAsFileObject if TYPE_CHECKING: # pragma: no cover from ..web_api import WebApi @@ -18,38 +15,127 @@ # work in progress -class DataPackageListContext(ABC): - @abstractmethod - def __enter__(self) -> "DataPackageListIterable": - ... +__pdoc__ = { + "DataPackageListContext.__init__": False, + "DataPackageListContextManager.__init__": False, +} - @abstractmethod - def __exit__(self, exception_type: Any, exception_value: Any, traceback: Any) -> None: - ... +class _DataPackageListContextIterator(Iterator[Tuple[str, datetime]], Iterable[Tuple[str, datetime]]): + _is_uesd = False + + def __init__(self, ijson_parse: Any) -> None: + self._ijson_parse = ijson_parse + + def __iter__(self) -> Iterator[Tuple[str, datetime]]: + if self._is_uesd: + raise Exception("iterator is already used") + self._is_uesd = True + return self + + def __next__(self) -> Tuple[str, datetime]: + name = "" + modified: Optional[datetime] = None -class DataPackageListIterable(ABC, Iterable[Tuple[str, datetime]]): + while True: + prefix, event, value = next(self._ijson_parse) + if event == "end_map": + if name == "": + raise Exception("bad format: name was not found") + if modified is None: + raise Exception("bad format: modified was not found") + return (name, modified) + + if event == "end_array": + raise StopIteration() + + if prefix == "entities.item.name": + if event != "string": + raise Exception("bad format: entities.item.name is not a string") + name = value + elif prefix == "entities.item.modified": + if event != "string": + raise Exception("bad format: entities.item.modified is not a string") + modified = _parse_iso8601(value) + + +class DataPackageListContext: @property - @abstractmethod - def body(self) -> DataPackageBody: - """_""" + def time_stamp_for_if_modified_since(self) -> datetime: + """ + A timestamp to pass as the ifModifiedSince parameter + in the next request to get incremental updates. + """ + return self._time_stamp_for_if_modified_since + @property + def download_full_list_on_or_after(self) -> Optional[datetime]: + """ + Recommended earliest next time to request a full list + by omitting timeStampForIfModifiedSince. + """ + return self._download_full_list_on_or_after -class _DataPackageListContext(DataPackageListContext, DataPackageListIterable, Iterator[Tuple[str, datetime]]): - response_: Optional["Response"] - _ijson_parse: Any - _body: Optional[DataPackageBody] + @property + def state(self) -> DataPackageListState: + """ + The state of this list. + """ + return self._state @property - def body(self) -> DataPackageBody: - return cast(DataPackageBody, self._body) + def items(self) -> Iterable[Tuple[str, datetime]]: + """An iterable contining tuples with the name and Timestamp when this entity was last modified""" + return self._items + + def __init__( + self, + time_stamp_for_if_modified_since: datetime, + download_full_list_on_or_after: Optional[datetime], + state: DataPackageListState, + items: _DataPackageListContextIterator, + ) -> None: + self._time_stamp_for_if_modified_since = time_stamp_for_if_modified_since + self._download_full_list_on_or_after = download_full_list_on_or_after + self._state = state + self._items = items + + +def _pars_body( + ijson_parse: Any, +) -> Tuple[Optional[datetime], Optional[datetime], Optional[DataPackageListState]]: + time_stamp_for_if_modified_since: Optional[datetime] = None + download_full_list_on_or_after: Optional[datetime] = None + state: Optional[DataPackageListState] = None + for prefix, event, value in ijson_parse: + if prefix == "timeStampForIfModifiedSince": + if event != "string": + raise Exception("bad format: timeStampForIfModifiedSince is not a string") + time_stamp_for_if_modified_since = _parse_iso8601(value) + elif prefix == "downloadFullListOnOrAfter": + if event != "string": + raise Exception("bad format: downloadFullListOnOrAfter is not a string") + download_full_list_on_or_after = _parse_iso8601(value) + elif prefix == "state": + if event != "number": + raise Exception("bad format: state is not a number") + state = DataPackageListState(value) + elif event == "start_array": + if prefix != "entities": + raise Exception("bad format: event start_array do not have a prefix of entities") + break + return time_stamp_for_if_modified_since, download_full_list_on_or_after, state + + +class DataPackageListContextManager: + _response: Optional["Response"] def __init__(self, if_modified_since: Optional[datetime], webApi: "WebApi") -> None: self._if_modified_since = if_modified_since self._webApi: Optional["WebApi"] = webApi self._iterator_started = False - def __enter__(self) -> "_DataPackageListContext": + def __enter__(self) -> DataPackageListContext: params = {} if self._if_modified_since: params["ifModifiedSince"] = self._if_modified_since.isoformat() @@ -58,14 +144,31 @@ def __enter__(self) -> "_DataPackageListContext": raise Exception("obj is closed") try: - self.response_ = self._webApi._session.get("v1/series/getdatapackagelist", params=params, stream=True) + self._response = self._webApi._session.get("v1/series/getdatapackagelist", params=params, stream=True) self._webApi = None - _raise_on_error(self.response_) - self._ijson_parse = ijson.parse(_ResponseAsFileObject(self.response_)) - self._set_body() - - return self + _raise_on_error(self._response) + ijson_parse = ijson.parse(_ResponseAsFileObject(self._response)) + + ( + time_stamp_for_if_modified_since, + download_full_list_on_or_after, + state, + ) = _pars_body(ijson_parse) + + if state is None: + raise Exception("bad format: state was not found") + if time_stamp_for_if_modified_since is None: + raise Exception("bad format: timeStampForIfModifiedSince was not found") + if not self._if_modified_since and download_full_list_on_or_after is None: + raise Exception("bad format: downloadFullListOnOrAfter was not found") + + return DataPackageListContext( + time_stamp_for_if_modified_since, + download_full_list_on_or_after, + state, + _DataPackageListContextIterator(ijson_parse), + ) except Exception as e: self.__exit__(None, None, None) @@ -73,68 +176,6 @@ def __enter__(self) -> "_DataPackageListContext": def __exit__(self, exception_type: Any, exception_value: Any, traceback: Any) -> None: self._webApi = None - if self.response_: - self.response_.close() - self.response_ = None - - def _set_body(self) -> None: - time_stamp_for_if_modified_since: Optional[datetime] = None - download_full_list_on_or_after: Optional[datetime] = None - state: Optional[DataPackageListState] = None - for prefix, event, value in self._ijson_parse: - if prefix == "timeStampForIfModifiedSince": - if event != "string": - raise Exception("bad format: timeStampForIfModifiedSince is not a string") - time_stamp_for_if_modified_since = _parse_iso8601(value) - elif prefix == "downloadFullListOnOrAfter": - if event != "string": - raise Exception("bad format: downloadFullListOnOrAfter is not a string") - download_full_list_on_or_after = _parse_iso8601(value) - elif prefix == "state": - if event != "number": - raise Exception("bad format: state is not a number") - state = DataPackageListState(value) - elif event == "start_array": - if prefix != "entities": - raise Exception("bad format: event start_array do not have a prefix of entities") - break - - if state is None: - raise Exception("bad format: state was not found") - if time_stamp_for_if_modified_since is None: - raise Exception("bad format: timeStampForIfModifiedSince was not found") - if not self._if_modified_since and download_full_list_on_or_after is None: - raise Exception("bad format: downloadFullListOnOrAfter was not found") - - self._body = DataPackageBody(time_stamp_for_if_modified_since, download_full_list_on_or_after, state) - - def __iter__(self) -> "_DataPackageListContext": - if self._iterator_started: - raise Exception("iterator has already started") - self._iterator_started = True - return self - - def __next__(self) -> Tuple[str, datetime]: - name = "" - modified: Optional[datetime] = None - - while True: - prefix, event, value = next(self._ijson_parse) - if event == "end_map": - if name == "": - raise Exception("bad format: name was not found") - if modified is None: - raise Exception("bad format: modified was not found") - return (name, modified) - - if event == "end_array": - raise StopIteration() - - if prefix == "entities.item.name": - if event != "string": - raise Exception("bad format: entities.item.name is not a string") - name = value - elif prefix == "entities.item.modified": - if event != "string": - raise Exception("bad format: entities.item.modified is not a string") - modified = _parse_iso8601(value) + if self._response: + self._response.close() + self._response = None From 302f7a2e5ff94f43fb65765081fed09f5de5456d Mon Sep 17 00:00:00 2001 From: mb-jp Date: Tue, 17 Oct 2023 15:18:40 +0200 Subject: [PATCH 03/12] wip --- macrobond_data_api/web/_web_api_revision.py | 6 +- macrobond_data_api/web/_web_only_api.py | 28 ++++++-- macrobond_data_api/web/session.py | 46 +++++++------ macrobond_data_api/web/web_types/__init__.py | 2 + .../web_types/data_package_list_context.py | 12 ++-- setup.py | 22 +++--- tests/Web/web_get_data_package_list.py | 52 ++++++++++++++ .../web_get_data_package_list_iterative.py | 67 +++++++++++++++++++ .../web_get_data_package_list_iterative_2.py | 59 ++++++++++++++++ 9 files changed, 244 insertions(+), 50 deletions(-) create mode 100644 tests/Web/web_get_data_package_list.py create mode 100644 tests/Web/web_get_data_package_list_iterative.py create mode 100644 tests/Web/web_get_data_package_list_iterative_2.py diff --git a/macrobond_data_api/web/_web_api_revision.py b/macrobond_data_api/web/_web_api_revision.py index 8b43e97f..da5cba13 100644 --- a/macrobond_data_api/web/_web_api_revision.py +++ b/macrobond_data_api/web/_web_api_revision.py @@ -20,7 +20,7 @@ from macrobond_data_api.common.types._repr_html_sequence import _ReprHtmlSequence from ._split_in_to_chunks import split_in_to_chunks -from .session import ProblemDetailsException, Session, _raise_on_error, _ResponseAsFileObject +from .session import ProblemDetailsException, Session if TYPE_CHECKING: # pragma: no cover from .web_api import WebApi @@ -274,8 +274,8 @@ def get_many_series_with_revisions( with self.session.series.post_fetch_all_vintage_series( _create_web_revision_h_request(requests_chunkd), stream=True ) as response: - _raise_on_error(response) - ijson_items = ijson.items(_ResponseAsFileObject(response), "item") + self.session.raise_on_error(response) + ijson_items = ijson.items(self.session.response_to_file_object(response), "item") item: "SeriesWithVintagesResponse" for item in ijson_items: error_code = item.get("errorCode") diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index 808d4508..2eece0ed 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -5,14 +5,13 @@ from macrobond_data_api.common.types import SearchResultLong from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 -from .web_types.data_package_list_context import DataPackageListContextManager +from .web_types.data_package_list_context import DataPackageListContextManager from .web_types.data_package_list_state import DataPackageListState from .web_types.data_pacakge_list_item import DataPackageListItem from .web_types.data_package_list import DataPackageList from .web_types.data_package_body import DataPackageBody -from .session import _raise_on_error, _ResponseAsFileObject from .subscription_list import SubscriptionList if TYPE_CHECKING: # pragma: no cover @@ -160,9 +159,8 @@ def get_data_package_list_iterative( if if_modified_since: params["ifModifiedSince"] = if_modified_since.isoformat() - with self._session.get("v1/series/getdatapackagelist", params=params, stream=True) as response: - _raise_on_error(response) - ijson_parse = ijson.parse(_ResponseAsFileObject(response)) + with self._session.get_or_raise("v1/series/getdatapackagelist", params=params, stream=True) as response: + ijson_parse = ijson.parse(self.session.response_to_file_object(response)) ( time_stamp_for_if_modified_since, @@ -187,10 +185,28 @@ def get_data_package_list_iterative( return body -# i need a good name ! +# TODO i need a good name ! def _get_data_package_list_iterative_2( self: "WebApi", if_modified_since: datetime = None ) -> DataPackageListContextManager: + # pylint: disable=line-too-long + """ + Process the data package list in batche. + This is more efficient since the complete list does not have to be in memory. + + Typically you want to pass the date of time_stamp_for_if_modified_since from response of the previous call + to get incremental updates. + + Parameters + ---------- + if_modified_since : datetime + The timestamp of the property time_stamp_for_if_modified_since from the response of the previous call. + If not specified, all items will be returned. + Returns + ------- + `macrobond_data_api.web.web_types.data_package_list_context.DataPackageListContextManager` + """ + # pylint: enable=line-too-long return DataPackageListContextManager(if_modified_since, self) diff --git a/macrobond_data_api/web/session.py b/macrobond_data_api/web/session.py index 5d9fc622..2f40906f 100644 --- a/macrobond_data_api/web/session.py +++ b/macrobond_data_api/web/session.py @@ -35,25 +35,6 @@ } -def _raise_on_error(response: "Response", non_error_status: Sequence[int] = None) -> "Response": - if non_error_status is None: - non_error_status = [200] - - if response.status_code in non_error_status: - return response - - content_type = response.headers.get("Content-Type") - - if content_type in ["application/json; charset=utf-8", "application/json"]: - raise ProblemDetailsException.create_from_response(response) - - macrobond_status = response.headers.get("X-Macrobond-Status") - if macrobond_status: - raise ProblemDetailsException(response, detail=macrobond_status) - - raise HttpException(response) - - class _ResponseAsFileObject: def __init__(self, response: "Response", chunk_size: int = 65536) -> None: self.data = response.iter_content(chunk_size=chunk_size) @@ -181,7 +162,7 @@ def get_or_raise( non_error_status: Sequence[int] = None, stream: bool = False, ) -> "Response": - return _raise_on_error(self.get(url, params, stream=stream), non_error_status) + return self.raise_on_error(self.get(url, params, stream=stream), non_error_status) def post(self, url: str, params: dict = None, json: object = None, stream: bool = False) -> "Response": return self._request("POST", url, params, json, stream) @@ -194,7 +175,7 @@ def post_or_raise( non_error_status: Sequence[int] = None, stream: bool = False, ) -> "Response": - return _raise_on_error(self.post(url, params, json, stream=stream), non_error_status) + return self.raise_on_error(self.post(url, params, json, stream=stream), non_error_status) def delete(self, url: str, params: dict = None, stream: bool = False) -> "Response": return self._request("DELETE", url, params, None, stream) @@ -206,7 +187,28 @@ def delete_or_raise( non_error_status: Sequence[int] = None, stream: bool = False, ) -> "Response": - return _raise_on_error(self.delete(url, params, stream=stream), non_error_status) + return self.raise_on_error(self.delete(url, params, stream=stream), non_error_status) + + def raise_on_error(self, response: "Response", non_error_status: Sequence[int] = None) -> "Response": + if non_error_status is None: + non_error_status = [200] + + if response.status_code in non_error_status: + return response + + content_type = response.headers.get("Content-Type") + + if content_type in ["application/json; charset=utf-8", "application/json"]: + raise ProblemDetailsException.create_from_response(response) + + macrobond_status = response.headers.get("X-Macrobond-Status") + if macrobond_status: + raise ProblemDetailsException(response, detail=macrobond_status) + + raise HttpException(response) + + def response_to_file_object(self, response: "Response") -> _ResponseAsFileObject: + return _ResponseAsFileObject(response) def _request(self, method: str, url: str, params: Optional[dict], json: object, stream: bool) -> "Response": if not self._is_open: diff --git a/macrobond_data_api/web/web_types/__init__.py b/macrobond_data_api/web/web_types/__init__.py index 211bea0f..9326d9a5 100644 --- a/macrobond_data_api/web/web_types/__init__.py +++ b/macrobond_data_api/web/web_types/__init__.py @@ -58,3 +58,5 @@ from .series_request import SeriesRequest from .in_house_series_methods import InHouseSeriesMethods + +from .data_package_list_context import DataPackageListContext, DataPackageListContextManager diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py index 9396628f..65054225 100644 --- a/macrobond_data_api/web/web_types/data_package_list_context.py +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -5,16 +5,12 @@ from macrobond_data_api.common.types._parse_iso8601 import _parse_iso8601 -from ..web_types.data_package_list_state import DataPackageListState -from ..session import _raise_on_error, _ResponseAsFileObject +from .data_package_list_state import DataPackageListState if TYPE_CHECKING: # pragma: no cover from ..web_api import WebApi from requests import Response -# work in progress - - __pdoc__ = { "DataPackageListContext.__init__": False, "DataPackageListContextManager.__init__": False, @@ -144,11 +140,11 @@ def __enter__(self) -> DataPackageListContext: raise Exception("obj is closed") try: - self._response = self._webApi._session.get("v1/series/getdatapackagelist", params=params, stream=True) + session = self._webApi._session self._webApi = None + self._response = session.get_or_raise("v1/series/getdatapackagelist", params=params, stream=True) - _raise_on_error(self._response) - ijson_parse = ijson.parse(_ResponseAsFileObject(self._response)) + ijson_parse = ijson.parse(session.response_to_file_object(self._response)) ( time_stamp_for_if_modified_since, diff --git a/setup.py b/setup.py index 790e6e0c..8cb4a4d1 100644 --- a/setup.py +++ b/setup.py @@ -85,22 +85,22 @@ extras_require={ "extra": ["matplotlib", "statsmodels", "scikit-learn", "pandas"], "dev": [ - "mypy==1.4.1", - "pylint==2.17.5", - "pycodestyle==2.10.0", + "mypy==1.6.0", + "pylint==3.0.1", + "pycodestyle==2.11.1", "pdoc3==0.10.0", - "build>=0.10.0", - "pytest==7.4.0", + "build>=1.0.3", + "pytest==7.4.2", "pytest-xdist==3.3.1", - "coverage>=7.2.7", - "black[jupyter]==23.7.0", + "coverage>=7.3.2", + "black[jupyter]==23.9.1", "requests[socks]>=2.31.0", "nbconvert==7.3.0", "ipython>=7.34.0", - "types-pywin32==305.0.0.10", - "types-requests==2.31.0.2", - "types-setuptools==68.0.0.3", - "filelock==3.12.2", + "types-pywin32==306.0.0.5", + "types-requests==2.31.0.9", + "types-setuptools==68.2.0.0", + "filelock==3.12.4", ], "socks": ["requests[socks]>=2.31.0"], }, diff --git a/tests/Web/web_get_data_package_list.py b/tests/Web/web_get_data_package_list.py new file mode 100644 index 00000000..0831edcb --- /dev/null +++ b/tests/Web/web_get_data_package_list.py @@ -0,0 +1,52 @@ +from datetime import datetime +from json import dumps as json_dumps +from typing import Any + +from requests import Response + +import pytest + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + def __init__(self, content: bytes): + self.content = content + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = Response() + response.status_code = 200 + response._content = self.content + return response + + +@pytest.mark.parametrize( + "state", + [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], +) +def test(state: DataPackageListState) -> None: + json = json_dumps( + { + "downloadFullListOnOrAfter": "2000-02-01T04:05:06", + "timeStampForIfModifiedSince": "2000-02-02T04:05:06", + "state": state, + "entities": [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + ], + } + ) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) + + r = api.get_data_package_list() + + assert r.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert r.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert r.state == state + assert r.items == [ + DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6)), + DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6)), + ] diff --git a/tests/Web/web_get_data_package_list_iterative.py b/tests/Web/web_get_data_package_list_iterative.py new file mode 100644 index 00000000..4d7d2f53 --- /dev/null +++ b/tests/Web/web_get_data_package_list_iterative.py @@ -0,0 +1,67 @@ +from datetime import datetime +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, List + +import pytest + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + def __init__(self, content: bytes): + self.content = content + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = Response() + response.status_code = 200 + response.raw = BytesIO(self.content) + return response + + +@pytest.mark.parametrize( + "state", + [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], +) +def test(state: DataPackageListState) -> None: + json = json_dumps( + { + "downloadFullListOnOrAfter": "2000-02-01T04:05:06", + "timeStampForIfModifiedSince": "2000-02-02T04:05:06", + "state": state, + "entities": [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + ], + } + ) + + hitponts = 2 + + def body_callback(body: DataPackageBody) -> None: + assert body.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert body.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert body.state == state + nonlocal hitponts + hitponts -= 1 + + def items_callback(body: DataPackageBody, data: List[DataPackageListItem]) -> None: + assert body.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert body.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert body.state == state + assert data == [ + DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6)), + DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6)), + ] + nonlocal hitponts + hitponts -= 1 + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) + + api.get_data_package_list_iterative(body_callback, items_callback) + + assert hitponts == 0 diff --git a/tests/Web/web_get_data_package_list_iterative_2.py b/tests/Web/web_get_data_package_list_iterative_2.py new file mode 100644 index 00000000..00127f88 --- /dev/null +++ b/tests/Web/web_get_data_package_list_iterative_2.py @@ -0,0 +1,59 @@ +from datetime import datetime +from io import BytesIO +from json import dumps as json_dumps +from typing import Any + +import pytest + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageListState + + +class TestAuth2Session: + def __init__(self, content: bytes): + self.content = content + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = Response() + response.status_code = 200 + response.raw = BytesIO(self.content) + return response + + +@pytest.mark.parametrize( + "state", + [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], +) +def test(state: DataPackageListState) -> None: + json = json_dumps( + { + "downloadFullListOnOrAfter": "2000-02-01T04:05:06", + "timeStampForIfModifiedSince": "2000-02-02T04:05:06", + "state": state, + "entities": [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + ], + } + ) + + hitponts = 1 + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) + + with api._get_data_package_list_iterative_2() as context: + assert context.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert context.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert context.state == state + + assert list(context.items) == [ + ("sek", datetime(2000, 2, 3, 4, 5, 6)), + ("dkk", datetime(2000, 2, 4, 4, 5, 6)), + ] + + hitponts -= 1 + + assert hitponts == 0 From aa4fd348947600cea0992268e68c41001cf6d3ea Mon Sep 17 00:00:00 2001 From: mb-jp Date: Thu, 19 Oct 2023 15:56:57 +0200 Subject: [PATCH 04/12] wip --- macrobond_data_api/web/_web_only_api.py | 19 +++++--- macrobond_data_api/web/web_api.py | 4 +- .../web_types/data_package_list_context.py | 40 ++++++++++------- ...y => web_get_data_package_list_chunked.py} | 45 +++++++++++++++---- 4 files changed, 77 insertions(+), 31 deletions(-) rename tests/Web/{web_get_data_package_list_iterative_2.py => web_get_data_package_list_chunked.py} (58%) diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index 2eece0ed..6206e5a7 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, List, Optional, Callable, Tuple +import warnings import ijson @@ -153,6 +154,12 @@ def get_data_package_list_iterative( `macrobond_data_api.web.web_types.data_package_body.DataPackageBody` """ # pylint: enable=line-too-long + warnings.warn( + "get_data_package_list_iterative is deprecated, Use get_data_package_list_chunked insted.", + DeprecationWarning, + 2, + ) + params = {} body: Optional[DataPackageBody] = None @@ -185,13 +192,12 @@ def get_data_package_list_iterative( return body -# TODO i need a good name ! -def _get_data_package_list_iterative_2( - self: "WebApi", if_modified_since: datetime = None +def get_data_package_list_chunked( + self: "WebApi", if_modified_since: datetime = None, chunk_size: int = 200 ) -> DataPackageListContextManager: # pylint: disable=line-too-long """ - Process the data package list in batche. + Process the data package list in chunkes. This is more efficient since the complete list does not have to be in memory. Typically you want to pass the date of time_stamp_for_if_modified_since from response of the previous call @@ -202,12 +208,15 @@ def _get_data_package_list_iterative_2( if_modified_since : datetime The timestamp of the property time_stamp_for_if_modified_since from the response of the previous call. If not specified, all items will be returned. + + chunk_size : int + The maximum number of items to include in each List in DataPackageListContext.items Returns ------- `macrobond_data_api.web.web_types.data_package_list_context.DataPackageListContextManager` """ # pylint: enable=line-too-long - return DataPackageListContextManager(if_modified_since, self) + return DataPackageListContextManager(if_modified_since, chunk_size, self) # Search diff --git a/macrobond_data_api/web/web_api.py b/macrobond_data_api/web/web_api.py index e7a60b16..77af3f66 100644 --- a/macrobond_data_api/web/web_api.py +++ b/macrobond_data_api/web/web_api.py @@ -4,7 +4,7 @@ entity_search_multi_filter_long, get_data_package_list, get_data_package_list_iterative, - _get_data_package_list_iterative_2, + get_data_package_list_chunked, subscription_list, ) from ._web_api_metadata import metadata_list_values, metadata_get_attribute_information, metadata_get_value_information @@ -72,7 +72,7 @@ def session(self) -> Session: get_data_package_list = get_data_package_list get_data_package_list_iterative = get_data_package_list_iterative - _get_data_package_list_iterative_2 = _get_data_package_list_iterative_2 + get_data_package_list_chunked = get_data_package_list_chunked entity_search_multi_filter_long = entity_search_multi_filter_long subscription_list = subscription_list diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py index 65054225..84a87da2 100644 --- a/macrobond_data_api/web/web_types/data_package_list_context.py +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import TYPE_CHECKING, Any, Optional, Tuple, Iterable, Iterator +from typing import TYPE_CHECKING, Any, Optional, Tuple, Iterable, Iterator, List import ijson @@ -17,22 +17,26 @@ } -class _DataPackageListContextIterator(Iterator[Tuple[str, datetime]], Iterable[Tuple[str, datetime]]): +class _DataPackageListContextIterator(Iterator[List[Tuple[str, datetime]]], Iterable[List[Tuple[str, datetime]]]): _is_uesd = False + _reached_the_end_of_array = False - def __init__(self, ijson_parse: Any) -> None: + def __init__(self, ijson_parse: Any, chunk_size: int) -> None: self._ijson_parse = ijson_parse + self.chunk_size = chunk_size - def __iter__(self) -> Iterator[Tuple[str, datetime]]: + def __iter__(self) -> Iterator[List[Tuple[str, datetime]]]: if self._is_uesd: raise Exception("iterator is already used") self._is_uesd = True return self - def __next__(self) -> Tuple[str, datetime]: + def __next__(self) -> List[Tuple[str, datetime]]: + if self._reached_the_end_of_array: + raise StopIteration() name = "" modified: Optional[datetime] = None - + items: List[Tuple[str, datetime]] = [] while True: prefix, event, value = next(self._ijson_parse) if event == "end_map": @@ -40,12 +44,17 @@ def __next__(self) -> Tuple[str, datetime]: raise Exception("bad format: name was not found") if modified is None: raise Exception("bad format: modified was not found") - return (name, modified) - - if event == "end_array": + items.append((name, modified)) + name = "" + modified = None + if len(items) == self.chunk_size: + return items + elif event == "end_array": + self._reached_the_end_of_array = True + if len(items) != 0: + return items raise StopIteration() - - if prefix == "entities.item.name": + elif prefix == "entities.item.name": if event != "string": raise Exception("bad format: entities.item.name is not a string") name = value @@ -80,8 +89,8 @@ def state(self) -> DataPackageListState: return self._state @property - def items(self) -> Iterable[Tuple[str, datetime]]: - """An iterable contining tuples with the name and Timestamp when this entity was last modified""" + def items(self) -> Iterable[List[Tuple[str, datetime]]]: + """An iterable contining Lists of tuples with the name and Timestamp when this entity was last modified""" return self._items def __init__( @@ -126,8 +135,9 @@ def _pars_body( class DataPackageListContextManager: _response: Optional["Response"] - def __init__(self, if_modified_since: Optional[datetime], webApi: "WebApi") -> None: + def __init__(self, if_modified_since: Optional[datetime], chunk_size: int, webApi: "WebApi") -> None: self._if_modified_since = if_modified_since + self.chunk_size = chunk_size self._webApi: Optional["WebApi"] = webApi self._iterator_started = False @@ -163,7 +173,7 @@ def __enter__(self) -> DataPackageListContext: time_stamp_for_if_modified_since, download_full_list_on_or_after, state, - _DataPackageListContextIterator(ijson_parse), + _DataPackageListContextIterator(ijson_parse, self.chunk_size), ) except Exception as e: diff --git a/tests/Web/web_get_data_package_list_iterative_2.py b/tests/Web/web_get_data_package_list_chunked.py similarity index 58% rename from tests/Web/web_get_data_package_list_iterative_2.py rename to tests/Web/web_get_data_package_list_chunked.py index 00127f88..cfbc85db 100644 --- a/tests/Web/web_get_data_package_list_iterative_2.py +++ b/tests/Web/web_get_data_package_list_chunked.py @@ -23,12 +23,8 @@ def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unu return response -@pytest.mark.parametrize( - "state", - [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], -) -def test(state: DataPackageListState) -> None: - json = json_dumps( +def get_json(state: DataPackageListState) -> str: + return json_dumps( { "downloadFullListOnOrAfter": "2000-02-01T04:05:06", "timeStampForIfModifiedSince": "2000-02-02T04:05:06", @@ -40,18 +36,49 @@ def test(state: DataPackageListState) -> None: } ) + +@pytest.mark.parametrize( + "state", + [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], +) +def test_1(state: DataPackageListState) -> None: + hitponts = 1 + json = get_json(state) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) + + with api.get_data_package_list_chunked() as context: + assert context.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert context.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert context.state == state + + assert list(context.items) == [ + [("sek", datetime(2000, 2, 3, 4, 5, 6)), ("dkk", datetime(2000, 2, 4, 4, 5, 6))], + ] + + hitponts -= 1 + + assert hitponts == 0 + + +@pytest.mark.parametrize( + "state", + [DataPackageListState.FULL_LISTING, DataPackageListState.INCOMPLETE, DataPackageListState.UP_TO_DATE], +) +def test_2(state: DataPackageListState) -> None: hitponts = 1 + json = get_json(state) api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) - with api._get_data_package_list_iterative_2() as context: + with api.get_data_package_list_chunked(chunk_size=1) as context: assert context.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) assert context.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) assert context.state == state assert list(context.items) == [ - ("sek", datetime(2000, 2, 3, 4, 5, 6)), - ("dkk", datetime(2000, 2, 4, 4, 5, 6)), + [("sek", datetime(2000, 2, 3, 4, 5, 6))], + [("dkk", datetime(2000, 2, 4, 4, 5, 6))], ] hitponts -= 1 From 64bc7722c19ceff55ebd47ffa3411ff0d449a955 Mon Sep 17 00:00:00 2001 From: Thomas Olsson <72909585+mb-to@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:15:44 +0200 Subject: [PATCH 05/12] Minor spelling --- macrobond_data_api/web/_web_only_api.py | 2 +- .../web/web_types/data_package_list_context.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index 6206e5a7..5c86b173 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -197,7 +197,7 @@ def get_data_package_list_chunked( ) -> DataPackageListContextManager: # pylint: disable=line-too-long """ - Process the data package list in chunkes. + Process the data package list in chunks. This is more efficient since the complete list does not have to be in memory. Typically you want to pass the date of time_stamp_for_if_modified_since from response of the previous call diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py index 84a87da2..e5cb042f 100644 --- a/macrobond_data_api/web/web_types/data_package_list_context.py +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -106,7 +106,7 @@ def __init__( self._items = items -def _pars_body( +def _parse_body( ijson_parse: Any, ) -> Tuple[Optional[datetime], Optional[datetime], Optional[DataPackageListState]]: time_stamp_for_if_modified_since: Optional[datetime] = None @@ -127,7 +127,7 @@ def _pars_body( state = DataPackageListState(value) elif event == "start_array": if prefix != "entities": - raise Exception("bad format: event start_array do not have a prefix of entities") + raise Exception("bad format: event start_array does not have a prefix of 'entities'") break return time_stamp_for_if_modified_since, download_full_list_on_or_after, state @@ -160,7 +160,7 @@ def __enter__(self) -> DataPackageListContext: time_stamp_for_if_modified_since, download_full_list_on_or_after, state, - ) = _pars_body(ijson_parse) + ) = _parse_body(ijson_parse) if state is None: raise Exception("bad format: state was not found") From 9965903cf7cbb2407832c7aaa35b37e28c89174a Mon Sep 17 00:00:00 2001 From: Thomas Olsson <72909585+mb-to@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:26:17 +0200 Subject: [PATCH 06/12] Added info in docs about deprecated method --- macrobond_data_api/web/_web_only_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index 5c86b173..c9db493f 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -129,6 +129,7 @@ def get_data_package_list_iterative( ) -> Optional[DataPackageBody]: # pylint: disable=line-too-long """ + .. Important:: This method is deprecated. Use `macrobond_data_api.web.web_api.WebApi.get_data_package_list_chunked` instead. Process the data package list in batches. This is more efficient since the complete list does not have to be in memory. @@ -155,7 +156,7 @@ def get_data_package_list_iterative( """ # pylint: enable=line-too-long warnings.warn( - "get_data_package_list_iterative is deprecated, Use get_data_package_list_chunked insted.", + "get_data_package_list_iterative is deprecated. Use get_data_package_list_chunked instead.", DeprecationWarning, 2, ) From bdb53f4554c1b2602d311301457d8805eb6e794d Mon Sep 17 00:00:00 2001 From: mb-jp Date: Fri, 20 Oct 2023 11:36:47 +0200 Subject: [PATCH 07/12] wip --- .../web/data_package_list_poller.py | 32 +- tests/Web/web_data_package_list_poller.py | 324 ++++++++++++++++++ tests/Web/web_get_data_package_list.py | 2 + .../Web/web_get_data_package_list_chunked.py | 2 + .../web_get_data_package_list_iterative.py | 7 +- 5 files changed, 354 insertions(+), 13 deletions(-) create mode 100644 tests/Web/web_data_package_list_poller.py diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index 65de51aa..cb7da4a3 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -2,7 +2,7 @@ from datetime import datetime, timezone import time -from typing import List, Optional, cast, TYPE_CHECKING, Callable +from typing import List, Optional, cast, TYPE_CHECKING from .web_api import WebApi from .web_types.data_package_list_state import DataPackageListState @@ -37,19 +37,23 @@ def __init__( api: WebApi, download_full_list_on_or_after: Optional[datetime] = None, time_stamp_for_if_modified_since: Optional[datetime] = None, - _sleep: Callable[[int], None] = time.sleep, + chunk_size: int = 200, ) -> None: + self._api = api + self._download_full_list_on_or_after = download_full_list_on_or_after + self._time_stamp_for_if_modified_since = time_stamp_for_if_modified_since + self._chunk_size = chunk_size + self.up_to_date_delay = 15 * 60 """ The time to wait, in seconds, between polls. """ self.incomplete_delay = 15 """ The time to wait, in seconds, between continuing partial updates. """ self.on_error_delay = 30 """ The time to wait, in seconds, before retrying after an error. """ - self._api = api - self._sleep = _sleep + + self._sleep = time.sleep + self._now = lambda: datetime.now(timezone.utc) self._abort = False - self._download_full_list_on_or_after = download_full_list_on_or_after - self._time_stamp_for_if_modified_since = time_stamp_for_if_modified_since @property def api(self) -> WebApi: @@ -77,8 +81,7 @@ def start(self) -> None: self._abort = False while not self._abort: if not self._time_stamp_for_if_modified_since or ( - self._download_full_list_on_or_after - and datetime.now(timezone.utc) > self._download_full_list_on_or_after + self._download_full_list_on_or_after and self._now() > self._download_full_list_on_or_after ): sub = self._run_full_listing() if sub: @@ -95,7 +98,7 @@ def start(self) -> None: self._sleep(self.up_to_date_delay) def _test_access(self) -> None: - params = {"ifModifiedSince": datetime(3000, 1, 1, tzinfo=timezone.utc)} + params = {"ifModifiedSince": datetime(3000, 1, 1, tzinfo=timezone.utc).isoformat()} response = self._api.session.get("v1/series/getdatapackagelist", params=params) if response.status_code == 403: raise Exception("Needs access - The account is not set up to use DataPackageList") @@ -104,7 +107,8 @@ def _run_full_listing(self, max_attempts: int = 3) -> Optional["DataPackageBody" is_stated = False def _body_callback(body: "DataPackageBody") -> None: - is_stated = True # pylint: disable=unused-variable + nonlocal is_stated + is_stated = True self.on_full_listing_start(body) try: @@ -114,6 +118,7 @@ def _body_callback(body: "DataPackageBody") -> None: _body_callback, self.on_full_listing_items, None, + self._chunk_size, ) if not sub: raise ValueError("subscription is None") @@ -124,7 +129,7 @@ def _body_callback(body: "DataPackageBody") -> None: if self._abort: raise _AbortException() from ex if attempt > max_attempts: - raise ex + raise self._sleep(self.on_error_delay) except _AbortException as ex: if is_stated: @@ -138,7 +143,8 @@ def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Op is_stated = False def _body_callback(body: "DataPackageBody") -> None: - is_stated = True # pylint: disable=unused-variable + nonlocal is_stated + is_stated = True self.on_incremental_start(body) try: @@ -148,6 +154,7 @@ def _body_callback(body: "DataPackageBody") -> None: _body_callback, self.on_incremental_items, if_modified_since, + self._chunk_size, ) break except Exception as ex: # pylint: disable=broad-except @@ -186,6 +193,7 @@ def _run_listing_incomplete( lambda _: None, self.on_incremental_items, if_modified_since, + self._chunk_size, ) if not sub: diff --git a/tests/Web/web_data_package_list_poller.py b/tests/Web/web_data_package_list_poller.py new file mode 100644 index 00000000..86c109f8 --- /dev/null +++ b/tests/Web/web_data_package_list_poller.py @@ -0,0 +1,324 @@ +from datetime import datetime, timezone +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, Dict, List, Optional + +import pytest + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + __test__ = False + + def __init__(self, content: List[bytes]): + self.index = 0 + self.content = content + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = Response() + response.status_code = 200 + response.raw = BytesIO(self.content[self.index]) + self.index += 1 + return response + + +def get_json( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", + entities: Optional[List[Dict[str, str]]] = None, +) -> str: + if entities is None: + entities = [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ] + return json_dumps( + { + "downloadFullListOnOrAfter": downloadFullListOnOrAfter, + "timeStampForIfModifiedSince": timeStampForIfModifiedSince, + "state": state, + "entities": entities, + } + ) + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + +def test_access() -> None: + hit = 0 + + def hit_test(now: int) -> None: + nonlocal hit + hit += 1 + assert hit == now + + class _TestAuth2Session: + __test__ = False + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + hit_test(1) + response = Response() + response.status_code = 403 + return response + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + api = WebApi(Session("", "", test_auth2_session=_TestAuth2Session())) + with pytest.raises(Exception, match="Needs access - The account is not set up to use DataPackageList"): + _TestDataPackageListPoller(api).start() + + assert hit == 1 + + +# _run_full_listing +def test_full_listing() -> None: + hit = 0 + + def hit_test(now: int) -> None: + nonlocal hit + hit += 1 + assert hit == now + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(7) + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.FULL_LISTING + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.FULL_LISTING + if hit == 3: + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + if hit == 4: + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + if hit == 5: + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(6) + assert is_aborted is False + assert exception is None + + json = get_json(DataPackageListState.FULL_LISTING) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller(api, chunk_size=1).start() + + assert hit == 7 + + +# _run_listing +def test_listing() -> None: + hit = 0 + + def hit_test(now: int) -> None: + nonlocal hit + hit += 1 + assert hit == now + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(8) + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.UP_TO_DATE + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.UP_TO_DATE + if hit == 4: + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 5: + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 6: + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + else: + raise Exception("should not be here") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is False + assert exception is None + + json = get_json(DataPackageListState.UP_TO_DATE) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + chunk_size=1, + ).start() + + assert hit == 8 + + +# _run_listing and _run_listing_incomplete +def test_listing_and_listing_incomplete() -> None: + hit = 0 + + def hit_test(now: int) -> None: + nonlocal hit + hit += 1 + assert hit == now + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + nonlocal hit + hit += 1 + if hit == 7: + assert secs == self.incomplete_delay + elif hit == 12: + assert secs == self.up_to_date_delay + raise Exception("End of test") + else: + raise Exception("should not be here") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + if hit == 3: + assert subscription.state == DataPackageListState.INCOMPLETE + else: + assert subscription.state == DataPackageListState.UP_TO_DATE + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + if hit == 4: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 5: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 6: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + elif hit == 8: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 9: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 10: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + else: + raise Exception("should not be here") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(11) + assert is_aborted is False + assert exception is None + + content = [ + bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), + bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), + ] + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + chunk_size=1, + ).start() + + assert hit == 12 diff --git a/tests/Web/web_get_data_package_list.py b/tests/Web/web_get_data_package_list.py index 0831edcb..90f5647b 100644 --- a/tests/Web/web_get_data_package_list.py +++ b/tests/Web/web_get_data_package_list.py @@ -12,6 +12,8 @@ class TestAuth2Session: + __test__ = False + def __init__(self, content: bytes): self.content = content diff --git a/tests/Web/web_get_data_package_list_chunked.py b/tests/Web/web_get_data_package_list_chunked.py index cfbc85db..9cf8d781 100644 --- a/tests/Web/web_get_data_package_list_chunked.py +++ b/tests/Web/web_get_data_package_list_chunked.py @@ -13,6 +13,8 @@ class TestAuth2Session: + __test__ = False + def __init__(self, content: bytes): self.content = content diff --git a/tests/Web/web_get_data_package_list_iterative.py b/tests/Web/web_get_data_package_list_iterative.py index 4d7d2f53..4adbcd4b 100644 --- a/tests/Web/web_get_data_package_list_iterative.py +++ b/tests/Web/web_get_data_package_list_iterative.py @@ -2,6 +2,7 @@ from io import BytesIO from json import dumps as json_dumps from typing import Any, List +import warnings import pytest @@ -13,6 +14,8 @@ class TestAuth2Session: + __test__ = False + def __init__(self, content: bytes): self.content = content @@ -62,6 +65,8 @@ def items_callback(body: DataPackageBody, data: List[DataPackageListItem]) -> No api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(bytes(json, "utf-8")))) - api.get_data_package_list_iterative(body_callback, items_callback) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + api.get_data_package_list_iterative(body_callback, items_callback) assert hitponts == 0 From ce990104cfb3b1f25e1aeb158ed49630b50b1283 Mon Sep 17 00:00:00 2001 From: mb-jp Date: Fri, 20 Oct 2023 16:27:47 +0200 Subject: [PATCH 08/12] wip --- .../web/data_package_list_poller.py | 118 +++--- tests/Web/web_data_package_list_poller.py | 344 ++++++++++++++++++ 2 files changed, 410 insertions(+), 52 deletions(-) diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index cb7da4a3..aa9f709b 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -2,13 +2,13 @@ from datetime import datetime, timezone import time -from typing import List, Optional, cast, TYPE_CHECKING +from typing import List, Optional, cast from .web_api import WebApi from .web_types.data_package_list_state import DataPackageListState -if TYPE_CHECKING: # pragma: no cover - from .web_types import DataPackageBody, DataPackageListItem +from .web_types.data_pacakge_list_item import DataPackageListItem +from .web_types.data_package_body import DataPackageBody class _AbortException(Exception): @@ -30,6 +30,8 @@ class DataPackageListPoller(ABC): The saved value of `download_full_list_on_or_after` from the previous run. `None` on first run. time_stamp_for_if_modified_since: datetime The saved value of `time_stamp_for_if_modified_since` from the previous run. `None`on first run. + chunk_size : int + The maximum number of items to include in each on_*_items() """ def __init__( @@ -103,28 +105,30 @@ def _test_access(self) -> None: if response.status_code == 403: raise Exception("Needs access - The account is not set up to use DataPackageList") - def _run_full_listing(self, max_attempts: int = 3) -> Optional["DataPackageBody"]: + def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: is_stated = False - def _body_callback(body: "DataPackageBody") -> None: - nonlocal is_stated - is_stated = True - self.on_full_listing_start(body) - try: for attempt in range(1, max_attempts): try: - sub = self._api.get_data_package_list_iterative( - _body_callback, - self.on_full_listing_items, - None, - self._chunk_size, - ) - if not sub: + body: DataPackageBody + with self._api.get_data_package_list_chunked(None, self._chunk_size) as context: + is_stated = True + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + self.on_full_listing_start(body) + for items in context.items: + self.on_full_listing_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + + if not body: raise ValueError("subscription is None") + is_stated = False self.on_full_listing_stop(False, None) - return sub + return body except Exception as ex: # pylint: disable=broad-except if self._abort: raise _AbortException() from ex @@ -139,23 +143,23 @@ def _body_callback(body: "DataPackageBody") -> None: self.on_full_listing_stop(False, ex) return None - def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional["DataPackageBody"]: + def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: is_stated = False - def _body_callback(body: "DataPackageBody") -> None: - nonlocal is_stated - is_stated = True - self.on_incremental_start(body) - try: + body: DataPackageBody for attempt in range(1, max_attempts): try: - sub = self._api.get_data_package_list_iterative( - _body_callback, - self.on_incremental_items, - if_modified_since, - self._chunk_size, - ) + with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: + is_stated = True + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + self.on_incremental_start(body) + for items in context.items: + self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) break except Exception as ex: # pylint: disable=broad-except if self._abort: @@ -164,16 +168,17 @@ def _body_callback(body: "DataPackageBody") -> None: raise self._sleep(self.on_error_delay) - if not sub: + if not body: raise ValueError("subscription is None") - if sub.state == DataPackageListState.UP_TO_DATE: + if body.state == DataPackageListState.UP_TO_DATE: + is_stated = False self.on_incremental_stop(False, None) - return sub + return body self._sleep(self.incomplete_delay) - return self._run_listing_incomplete(sub.time_stamp_for_if_modified_since, is_stated, max_attempts) + return self._run_listing_incomplete(body.time_stamp_for_if_modified_since, is_stated, max_attempts) except _AbortException as ex: if is_stated: self.on_incremental_stop(True, cast(Exception, ex.__cause__)) @@ -182,30 +187,36 @@ def _body_callback(body: "DataPackageBody") -> None: self.on_incremental_stop(False, ex) return None - def _run_listing_incomplete( + def _run_listing_incomplete( # pylint: disable=too-many-branches self, if_modified_since: datetime, is_stated: bool, max_attempts: int = 3 - ) -> Optional["DataPackageBody"]: + ) -> Optional[DataPackageBody]: try: while True: for attempt in range(1, max_attempts): try: - sub = self._api.get_data_package_list_iterative( - lambda _: None, - self.on_incremental_items, - if_modified_since, - self._chunk_size, - ) - - if not sub: + body: DataPackageBody + with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + for items in context.items: + self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + + if not body: raise ValueError("subscription is None") - if sub.state == DataPackageListState.UP_TO_DATE: - self.on_incremental_stop(False, None) - return sub + if body.state == DataPackageListState.UP_TO_DATE: + try: + self.on_incremental_stop(False, None) + except _AbortException: + ... + return body self._sleep(self.incomplete_delay) - if_modified_since = sub.time_stamp_for_if_modified_since + if_modified_since = body.time_stamp_for_if_modified_since except Exception as ex2: # pylint: disable=broad-except if self._abort: raise _AbortException() from ex2 @@ -223,11 +234,11 @@ def _run_listing_incomplete( # full_listing @abstractmethod - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_start(self, subscription: DataPackageBody) -> None: """This override is called when a full listing starts.""" @abstractmethod - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: """This override is called repeatedly with one or more items until all items are listed.""" @abstractmethod @@ -245,11 +256,11 @@ def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) # listing @abstractmethod - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_start(self, subscription: DataPackageBody) -> None: """This override is called when an incremental listing starts.""" @abstractmethod - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: """This override is called repeatedly with one or more items until all updated items are listed.""" @abstractmethod @@ -265,5 +276,8 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) """ def abort(self) -> None: - """Call this method to stop processing.""" + """ + Call this method to stop processing. + """ self._abort = True + raise _AbortException() diff --git a/tests/Web/web_data_package_list_poller.py b/tests/Web/web_data_package_list_poller.py index 86c109f8..c4099565 100644 --- a/tests/Web/web_data_package_list_poller.py +++ b/tests/Web/web_data_package_list_poller.py @@ -322,3 +322,347 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) ).start() assert hit == 12 + + +# test_abort_full_listing + + +def test_abort_full_listing_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + self.abort() + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(3) + assert is_aborted is True + assert exception + + json = get_json(DataPackageListState.FULL_LISTING) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller(api, chunk_size=1).start() + + assert hit == 3 + + +def test_abort_full_listing_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + + def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + if hit_test(3, 4) == 4: + self.abort() + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + assert is_aborted is True + assert exception + + json = get_json(DataPackageListState.FULL_LISTING) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller(api, chunk_size=1).start() + + assert hit == 5 + + +def test_abort_full_listing_3() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + + def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(3) + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(4) + assert is_aborted is False + assert exception is None + self.abort() + + json = get_json(DataPackageListState.FULL_LISTING) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller(api).start() + + assert hit == 4 + + +# test_abort_listing + + +def test_abort_listing_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + self.abort() + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(4) + assert is_aborted is True + assert exception is not None + + json = get_json(DataPackageListState.UP_TO_DATE) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 4 + + +def test_abort_listing_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + if hit_test(4): + self.abort() + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + assert is_aborted is True + assert exception is not None + + json = get_json(DataPackageListState.UP_TO_DATE) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + chunk_size=1, + ).start() + + assert hit == 5 + + +def test_abort_listing_3() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(4) + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + assert is_aborted is False + assert exception is None + self.abort() + + json = get_json(DataPackageListState.UP_TO_DATE) + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 5 + + +# test_abort_listing_and_listing_incomplete + + +def test_abort_listing_and_listing_incomplete_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(5) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + if hit_test(4, 6) == 6: + self.abort() + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is True + assert exception is not None + + content = [ + bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), + bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), + ] + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 7 + + +def test_abort_listing_and_listing_incomplete_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(5) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4, 6) + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is False + assert exception is None + self.abort() + + content = [ + bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), + bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), + ] + + api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 7 From 9db5da66eff54861097926b080050bd9d75b5d5c Mon Sep 17 00:00:00 2001 From: mb-jp Date: Mon, 23 Oct 2023 11:09:32 +0200 Subject: [PATCH 09/12] wip --- macrobond_data_api/com/_metadata_directory.py | 2 +- macrobond_data_api/com/com_client.py | 2 +- .../common/types/_repr_html_sequence.py | 4 +- .../types/get_all_vintage_series_result.py | 4 +- .../types/metadata_value_information.py | 4 +- .../common/types/search_result.py | 4 +- .../common/types/search_result_long.py | 4 +- .../common/types/start_or_end_point.py | 4 +- .../common/types/unified_series.py | 4 +- .../util/transfer_performance_test.py | 2 +- macrobond_data_api/web/_web_only_api.py | 6 +- .../web/data_package_list_poller.py | 223 +++++---- .../web/web_types/data_package_list.py | 4 +- .../abort.py} | 314 ++----------- tests/Web/data_package_list_poller/error.py | 430 ++++++++++++++++++ tests/Web/data_package_list_poller/normal.py | 292 ++++++++++++ tests/Web/data_package_list_poller/retry.py | 239 ++++++++++ .../data_package_list_poller/test_access.py | 91 ++++ 18 files changed, 1214 insertions(+), 419 deletions(-) rename tests/Web/{web_data_package_list_poller.py => data_package_list_poller/abort.py} (50%) create mode 100644 tests/Web/data_package_list_poller/error.py create mode 100644 tests/Web/data_package_list_poller/normal.py create mode 100644 tests/Web/data_package_list_poller/retry.py create mode 100644 tests/Web/data_package_list_poller/test_access.py diff --git a/macrobond_data_api/com/_metadata_directory.py b/macrobond_data_api/com/_metadata_directory.py index 2c515cf5..550f8557 100644 --- a/macrobond_data_api/com/_metadata_directory.py +++ b/macrobond_data_api/com/_metadata_directory.py @@ -4,7 +4,7 @@ try: from pywintypes import TimeType except ImportError as ex_: - ... + pass if TYPE_CHECKING: # pragma: no cover from .com_types.connection import Connection diff --git a/macrobond_data_api/com/com_client.py b/macrobond_data_api/com/com_client.py index 8ebf48af..93ad70de 100644 --- a/macrobond_data_api/com/com_client.py +++ b/macrobond_data_api/com/com_client.py @@ -25,7 +25,7 @@ # winreg is not available on linux so mypy will fail on build server as it is runiong on linux from winreg import OpenKey, QueryValueEx, HKEY_CLASSES_ROOT, HKEY_CURRENT_USER # type: ignore except ImportError: - ... + pass def _test_regedit_assembly() -> Optional[str]: diff --git a/macrobond_data_api/common/types/_repr_html_sequence.py b/macrobond_data_api/common/types/_repr_html_sequence.py index cadb2bd6..5c23c2c0 100644 --- a/macrobond_data_api/common/types/_repr_html_sequence.py +++ b/macrobond_data_api/common/types/_repr_html_sequence.py @@ -20,11 +20,11 @@ def __init__(self, items: Sequence[_TypeVar]) -> None: @overload def __getitem__(self, i: int) -> _TypeVar: - ... + pass @overload def __getitem__(self, s: slice) -> Sequence[_TypeVar]: - ... + pass def __getitem__(self, key): # type: ignore return _ReprHtmlSequence(self.items[key]) if isinstance(key, slice) else self.items[key] diff --git a/macrobond_data_api/common/types/get_all_vintage_series_result.py b/macrobond_data_api/common/types/get_all_vintage_series_result.py index 806df6a3..264fea64 100644 --- a/macrobond_data_api/common/types/get_all_vintage_series_result.py +++ b/macrobond_data_api/common/types/get_all_vintage_series_result.py @@ -64,11 +64,11 @@ def _repr_html_(self) -> str: @overload def __getitem__(self, i: int) -> VintageSeries: - ... + pass @overload def __getitem__(self, s: slice) -> Sequence[VintageSeries]: - ... + pass def __getitem__(self, key): # type: ignore return self.series[key] diff --git a/macrobond_data_api/common/types/metadata_value_information.py b/macrobond_data_api/common/types/metadata_value_information.py index e759ea61..f17e75ec 100644 --- a/macrobond_data_api/common/types/metadata_value_information.py +++ b/macrobond_data_api/common/types/metadata_value_information.py @@ -111,11 +111,11 @@ def to_dict(self) -> List[TypedDictMetadataValueInformationItem]: @overload def __getitem__(self, i: int) -> MetadataValueInformationItem: - ... + pass @overload def __getitem__(self, s: slice) -> List[MetadataValueInformationItem]: - ... + pass def __getitem__(self, key): # type: ignore return self.entities[key] diff --git a/macrobond_data_api/common/types/search_result.py b/macrobond_data_api/common/types/search_result.py index 40d9ff62..4bf3f1fb 100644 --- a/macrobond_data_api/common/types/search_result.py +++ b/macrobond_data_api/common/types/search_result.py @@ -51,11 +51,11 @@ def _repr_html_(self) -> str: @overload def __getitem__(self, i: int) -> "Metadata": - ... + pass @overload def __getitem__(self, s: slice) -> Sequence["Metadata"]: - ... + pass def __getitem__(self, key): # type: ignore return self.entities[key] diff --git a/macrobond_data_api/common/types/search_result_long.py b/macrobond_data_api/common/types/search_result_long.py index 6622aad3..628e4e44 100644 --- a/macrobond_data_api/common/types/search_result_long.py +++ b/macrobond_data_api/common/types/search_result_long.py @@ -48,11 +48,11 @@ def _repr_html_(self) -> str: @overload def __getitem__(self, i: int) -> str: - ... + pass @overload def __getitem__(self, s: slice) -> List[str]: - ... + pass def __getitem__(self, key): # type: ignore return self.entities[key] diff --git a/macrobond_data_api/common/types/start_or_end_point.py b/macrobond_data_api/common/types/start_or_end_point.py index 6f4f89ad..a9dbc151 100644 --- a/macrobond_data_api/common/types/start_or_end_point.py +++ b/macrobond_data_api/common/types/start_or_end_point.py @@ -86,12 +86,12 @@ def point_in_time( mm: int = None, # pylint: disable = invalid-name dd: int = None, # pylint: disable = invalid-name ) -> "StartOrEndPoint": - ... + pass @overload @staticmethod def point_in_time(yyyy_or_datetime: datetime) -> "StartOrEndPoint": - ... + pass @staticmethod def point_in_time( diff --git a/macrobond_data_api/common/types/unified_series.py b/macrobond_data_api/common/types/unified_series.py index 42e66384..db1a9200 100644 --- a/macrobond_data_api/common/types/unified_series.py +++ b/macrobond_data_api/common/types/unified_series.py @@ -146,11 +146,11 @@ def _repr_html_(self) -> str: @overload def __getitem__(self, i: int) -> UnifiedSeries: - ... + pass @overload def __getitem__(self, s: slice) -> List[UnifiedSeries]: - ... + pass def __getitem__(self, key): # type: ignore return self.series[key] diff --git a/macrobond_data_api/util/transfer_performance_test.py b/macrobond_data_api/util/transfer_performance_test.py index 197f308a..ce51b04b 100644 --- a/macrobond_data_api/util/transfer_performance_test.py +++ b/macrobond_data_api/util/transfer_performance_test.py @@ -118,7 +118,7 @@ def run_integrity_tests(self, indicator: bool, times: int) -> None: for i in range(0, times): result = _Result.run_integrity_test(self.size_kB, i) if result.error is not None: - ... + pass # print(f" Error: {str(result.error)} ", end="", flush=True) elif indicator: print(".", end="", flush=True) diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index c9db493f..db502c85 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -92,7 +92,7 @@ def _get_data_package_list_iterative_pars_items( return True -def get_data_package_list(self: "WebApi", if_modified_since: datetime = None) -> DataPackageList: +def get_data_package_list(self: "WebApi", if_modified_since: Optional[datetime] = None) -> DataPackageList: # pylint: disable=line-too-long """ Get the items in the data package. @@ -124,7 +124,7 @@ def get_data_package_list_iterative( self: "WebApi", body_callback: Callable[[DataPackageBody], Optional[bool]], items_callback: Callable[[DataPackageBody, List[DataPackageListItem]], Optional[bool]], - if_modified_since: datetime = None, + if_modified_since: Optional[datetime] = None, buffer_size: int = 200, ) -> Optional[DataPackageBody]: # pylint: disable=line-too-long @@ -194,7 +194,7 @@ def get_data_package_list_iterative( def get_data_package_list_chunked( - self: "WebApi", if_modified_since: datetime = None, chunk_size: int = 200 + self: "WebApi", if_modified_since: Optional[datetime] = None, chunk_size: int = 200 ) -> DataPackageListContextManager: # pylint: disable=line-too-long """ diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index aa9f709b..fcdd13bf 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -2,7 +2,8 @@ from datetime import datetime, timezone import time -from typing import List, Optional, cast +from typing import List, Optional +import warnings from .web_api import WebApi from .web_types.data_package_list_state import DataPackageListState @@ -11,10 +12,6 @@ from .web_types.data_package_body import DataPackageBody -class _AbortException(Exception): - ... - - class DataPackageListPoller(ABC): """ This is work in progress and might change soon. @@ -91,6 +88,11 @@ def start(self) -> None: self._time_stamp_for_if_modified_since = sub.time_stamp_for_if_modified_since else: sub = self._run_listing(self._time_stamp_for_if_modified_since) + + if sub and sub.state != DataPackageListState.UP_TO_DATE: + self._sleep(self.incomplete_delay) + sub = self._run_listing_incomplete(sub.time_stamp_for_if_modified_since) + if sub: self._time_stamp_for_if_modified_since = sub.time_stamp_for_if_modified_since @@ -106,130 +108,124 @@ def _test_access(self) -> None: raise Exception("Needs access - The account is not set up to use DataPackageList") def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: - is_stated = False + is_running = False + attempt = 0 + while True: + attempt += 1 + try: + with self._api.get_data_package_list_chunked(None, self._chunk_size) as context: + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + is_running = True + self.on_full_listing_start(body) - try: - for attempt in range(1, max_attempts): - try: - body: DataPackageBody - with self._api.get_data_package_list_chunked(None, self._chunk_size) as context: - is_stated = True - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, - ) - self.on_full_listing_start(body) - for items in context.items: - self.on_full_listing_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) - - if not body: - raise ValueError("subscription is None") - - is_stated = False - self.on_full_listing_stop(False, None) - return body - except Exception as ex: # pylint: disable=broad-except if self._abort: - raise _AbortException() from ex - if attempt > max_attempts: - raise - self._sleep(self.on_error_delay) - except _AbortException as ex: - if is_stated: - self.on_full_listing_stop(True, cast(Exception, ex.__cause__)) + self._run_on_full_listing_stop(True, None) + return None + + for items in context.items: + self.on_full_listing_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + if self._abort: + self._run_on_full_listing_stop(True, None) + return None + + self._run_on_full_listing_stop(False, None) + + return body + except Exception as ex: # pylint: disable=broad-except + if is_running: + self._run_on_full_listing_stop(False, ex) + return None + if attempt > max_attempts: + raise + self._sleep(self.on_error_delay) + + def _run_on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + try: + self.on_full_listing_stop(is_aborted, exception) except Exception as ex: # pylint: disable=broad-except - if is_stated: - self.on_full_listing_stop(False, ex) - return None + warnings.warn("Unhandled exception in on_full_listing_stop, " + str(ex)) + self._sleep(self.on_error_delay) def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: is_stated = False + attempt = 0 + while True: + attempt += 1 + try: + with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + is_stated = True + self.on_incremental_start(body) - try: - body: DataPackageBody - for attempt in range(1, max_attempts): - try: - with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: - is_stated = True - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, - ) - self.on_incremental_start(body) - for items in context.items: - self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) - break - except Exception as ex: # pylint: disable=broad-except if self._abort: - raise _AbortException() from ex - if attempt > max_attempts: - raise - self._sleep(self.on_error_delay) + self._run_on_incremental_stop(True, None) + return None - if not body: - raise ValueError("subscription is None") + for items in context.items: + self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + if self._abort: + self._run_on_incremental_stop(True, None) + return None - if body.state == DataPackageListState.UP_TO_DATE: - is_stated = False - self.on_incremental_stop(False, None) + if body.state == DataPackageListState.UP_TO_DATE: + self._run_on_incremental_stop(False, None) return body + except Exception as ex: # pylint: disable=broad-except + if is_stated: + self._run_on_incremental_stop(False, ex) + return None + if attempt > max_attempts: + raise + self._sleep(self.on_error_delay) + + def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: + is_stated = False + attempt = 0 + while True: + attempt += 1 + try: + with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + is_stated = True + for items in context.items: + self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + if self._abort: + self._run_on_incremental_stop(True, None) + return None - self._sleep(self.incomplete_delay) + if body.state == DataPackageListState.UP_TO_DATE: + self._run_on_incremental_stop(False, None) + return body - return self._run_listing_incomplete(body.time_stamp_for_if_modified_since, is_stated, max_attempts) - except _AbortException as ex: - if is_stated: - self.on_incremental_stop(True, cast(Exception, ex.__cause__)) - except Exception as ex: # pylint: disable=broad-except - if is_stated: - self.on_incremental_stop(False, ex) - return None + self._sleep(self.incomplete_delay) - def _run_listing_incomplete( # pylint: disable=too-many-branches - self, if_modified_since: datetime, is_stated: bool, max_attempts: int = 3 - ) -> Optional[DataPackageBody]: + if_modified_since = body.time_stamp_for_if_modified_since + except Exception as ex: # pylint: disable=broad-except + if is_stated: + self._run_on_incremental_stop(False, ex) + return None + if attempt > max_attempts: + raise + self._sleep(self.on_error_delay) + + def _run_on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: try: - while True: - for attempt in range(1, max_attempts): - try: - body: DataPackageBody - with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, - ) - for items in context.items: - self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) - - if not body: - raise ValueError("subscription is None") - - if body.state == DataPackageListState.UP_TO_DATE: - try: - self.on_incremental_stop(False, None) - except _AbortException: - ... - return body - - self._sleep(self.incomplete_delay) - - if_modified_since = body.time_stamp_for_if_modified_since - except Exception as ex2: # pylint: disable=broad-except - if self._abort: - raise _AbortException() from ex2 - if attempt > max_attempts: - raise - self._sleep(self.on_error_delay) - except _AbortException as ex: - if is_stated: - self.on_incremental_stop(True, cast(Exception, ex.__cause__)) + self.on_incremental_stop(is_aborted, exception) except Exception as ex: # pylint: disable=broad-except - if is_stated: - self.on_incremental_stop(False, ex) - return None + warnings.warn("Unhandled exception in on_incremental_stop, " + str(ex)) + self._sleep(self.on_error_delay) # full_listing @@ -280,4 +276,3 @@ def abort(self) -> None: Call this method to stop processing. """ self._abort = True - raise _AbortException() diff --git a/macrobond_data_api/web/web_types/data_package_list.py b/macrobond_data_api/web/web_types/data_package_list.py index f0ac7d63..9fa4bcd0 100644 --- a/macrobond_data_api/web/web_types/data_package_list.py +++ b/macrobond_data_api/web/web_types/data_package_list.py @@ -30,11 +30,11 @@ def __init__(self, response: "FeedEntitiesResponse") -> None: @overload def __getitem__(self, i: int) -> DataPackageListItem: - ... + pass @overload def __getitem__(self, s: slice) -> List[DataPackageListItem]: - ... + pass def __getitem__(self, key): # type: ignore return self.items[key] diff --git a/tests/Web/web_data_package_list_poller.py b/tests/Web/data_package_list_poller/abort.py similarity index 50% rename from tests/Web/web_data_package_list_poller.py rename to tests/Web/data_package_list_poller/abort.py index c4099565..5ccf65e8 100644 --- a/tests/Web/web_data_package_list_poller.py +++ b/tests/Web/data_package_list_poller/abort.py @@ -3,8 +3,6 @@ from json import dumps as json_dumps from typing import Any, Dict, List, Optional -import pytest - from requests import Response from macrobond_data_api.web import WebApi @@ -16,31 +14,33 @@ class TestAuth2Session: __test__ = False - def __init__(self, content: List[bytes]): + def __init__(self, *responses: Response): self.index = 0 - self.content = content + self.responses = responses def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument - response = Response() - response.status_code = 200 - response.raw = BytesIO(self.content[self.index]) + response = self.responses[self.index] self.index += 1 return response -def get_json( +def get_api(*responses: Response) -> WebApi: + return WebApi(Session("", "", test_auth2_session=TestAuth2Session(*responses))) + + +def get_json_response( state: DataPackageListState, downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", entities: Optional[List[Dict[str, str]]] = None, -) -> str: +) -> Response: if entities is None: entities = [ {"name": "sek", "modified": "2000-02-03T04:05:06"}, {"name": "dkk", "modified": "2000-02-04T04:05:06"}, {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, ] - return json_dumps( + json = json_dumps( { "downloadFullListOnOrAfter": downloadFullListOnOrAfter, "timeStampForIfModifiedSince": timeStampForIfModifiedSince, @@ -48,6 +48,10 @@ def get_json( "entities": entities, } ) + response = Response() + response.status_code = 200 + response.raw = BytesIO(bytes(json, "utf-8")) + return response class TestDataPackageListPoller(DataPackageListPoller): @@ -89,244 +93,6 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) raise Exception("should not be called") -def test_access() -> None: - hit = 0 - - def hit_test(now: int) -> None: - nonlocal hit - hit += 1 - assert hit == now - - class _TestAuth2Session: - __test__ = False - - def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument - hit_test(1) - response = Response() - response.status_code = 403 - return response - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - api = WebApi(Session("", "", test_auth2_session=_TestAuth2Session())) - with pytest.raises(Exception, match="Needs access - The account is not set up to use DataPackageList"): - _TestDataPackageListPoller(api).start() - - assert hit == 1 - - -# _run_full_listing -def test_full_listing() -> None: - hit = 0 - - def hit_test(now: int) -> None: - nonlocal hit - hit += 1 - assert hit == now - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def sleep(self, secs: float) -> None: - hit_test(7) - assert secs == self.up_to_date_delay - raise Exception("End of test") - - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: - hit_test(2) - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - assert subscription.state == DataPackageListState.FULL_LISTING - - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: - nonlocal hit - hit += 1 - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - assert subscription.state == DataPackageListState.FULL_LISTING - if hit == 3: - assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] - if hit == 4: - assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] - if hit == 5: - assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(6) - assert is_aborted is False - assert exception is None - - json = get_json(DataPackageListState.FULL_LISTING) - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) - - with pytest.raises(Exception, match="End of test"): - _TestDataPackageListPoller(api, chunk_size=1).start() - - assert hit == 7 - - -# _run_listing -def test_listing() -> None: - hit = 0 - - def hit_test(now: int) -> None: - nonlocal hit - hit += 1 - assert hit == now - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def sleep(self, secs: float) -> None: - hit_test(8) - assert secs == self.up_to_date_delay - raise Exception("End of test") - - def now(self) -> datetime: - hit_test(2) - return datetime(2000, 1, 1, tzinfo=timezone.utc) - - def on_incremental_start(self, subscription: "DataPackageBody") -> None: - hit_test(3) - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - assert subscription.state == DataPackageListState.UP_TO_DATE - - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: - nonlocal hit - hit += 1 - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - assert subscription.state == DataPackageListState.UP_TO_DATE - if hit == 4: - assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] - elif hit == 5: - assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] - elif hit == 6: - assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - else: - raise Exception("should not be here") - - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(7) - assert is_aborted is False - assert exception is None - - json = get_json(DataPackageListState.UP_TO_DATE) - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) - - with pytest.raises(Exception, match="End of test"): - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - chunk_size=1, - ).start() - - assert hit == 8 - - -# _run_listing and _run_listing_incomplete -def test_listing_and_listing_incomplete() -> None: - hit = 0 - - def hit_test(now: int) -> None: - nonlocal hit - hit += 1 - assert hit == now - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def sleep(self, secs: float) -> None: - nonlocal hit - hit += 1 - if hit == 7: - assert secs == self.incomplete_delay - elif hit == 12: - assert secs == self.up_to_date_delay - raise Exception("End of test") - else: - raise Exception("should not be here") - - def now(self) -> datetime: - hit_test(2) - return datetime(2000, 1, 1, tzinfo=timezone.utc) - - def on_incremental_start(self, subscription: "DataPackageBody") -> None: - nonlocal hit - hit += 1 - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - if hit == 3: - assert subscription.state == DataPackageListState.INCOMPLETE - else: - assert subscription.state == DataPackageListState.UP_TO_DATE - - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: - nonlocal hit - hit += 1 - assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) - assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) - if hit == 4: - assert subscription.state == DataPackageListState.INCOMPLETE - assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] - elif hit == 5: - assert subscription.state == DataPackageListState.INCOMPLETE - assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] - elif hit == 6: - assert subscription.state == DataPackageListState.INCOMPLETE - assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - elif hit == 8: - assert subscription.state == DataPackageListState.UP_TO_DATE - assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] - elif hit == 9: - assert subscription.state == DataPackageListState.UP_TO_DATE - assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] - elif hit == 10: - assert subscription.state == DataPackageListState.UP_TO_DATE - assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - else: - raise Exception("should not be here") - - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(11) - assert is_aborted is False - assert exception is None - - content = [ - bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), - bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), - ] - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) - - with pytest.raises(Exception, match="End of test"): - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - chunk_size=1, - ).start() - - assert hit == 12 - - -# test_abort_full_listing - - def test_abort_full_listing_1() -> None: hit = 0 @@ -349,11 +115,9 @@ def on_full_listing_start(self, subscription: "DataPackageBody") -> None: def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(3) assert is_aborted is True - assert exception - - json = get_json(DataPackageListState.FULL_LISTING) + assert exception is None - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) _TestDataPackageListPoller(api, chunk_size=1).start() @@ -385,11 +149,9 @@ def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataP def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) assert is_aborted is True - assert exception - - json = get_json(DataPackageListState.FULL_LISTING) + assert exception is None - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) _TestDataPackageListPoller(api, chunk_size=1).start() @@ -423,9 +185,7 @@ def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) assert exception is None self.abort() - json = get_json(DataPackageListState.FULL_LISTING) - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) _TestDataPackageListPoller(api).start() @@ -461,11 +221,9 @@ def on_incremental_start(self, subscription: "DataPackageBody") -> None: def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(4) assert is_aborted is True - assert exception is not None - - json = get_json(DataPackageListState.UP_TO_DATE) + assert exception is None - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) _TestDataPackageListPoller( api, @@ -505,11 +263,9 @@ def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPa def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) assert is_aborted is True - assert exception is not None - - json = get_json(DataPackageListState.UP_TO_DATE) + assert exception is None - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) _TestDataPackageListPoller( api, @@ -552,9 +308,7 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) assert exception is None self.abort() - json = get_json(DataPackageListState.UP_TO_DATE) - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session([bytes(json, "utf-8")]))) + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) _TestDataPackageListPoller( api, @@ -600,14 +354,11 @@ def on_incremental_items(self, subscription: "DataPackageBody", items: List["Dat def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(7) assert is_aborted is True - assert exception is not None - - content = [ - bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), - bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), - ] + assert exception is None - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) + ) _TestDataPackageListPoller( api, @@ -652,12 +403,9 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) assert exception is None self.abort() - content = [ - bytes(get_json(DataPackageListState.INCOMPLETE), "utf-8"), - bytes(get_json(DataPackageListState.UP_TO_DATE), "utf-8"), - ] - - api = WebApi(Session("", "", test_auth2_session=TestAuth2Session(content))) + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) + ) _TestDataPackageListPoller( api, diff --git a/tests/Web/data_package_list_poller/error.py b/tests/Web/data_package_list_poller/error.py new file mode 100644 index 00000000..ff237b99 --- /dev/null +++ b/tests/Web/data_package_list_poller/error.py @@ -0,0 +1,430 @@ +from datetime import datetime, timezone +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, Dict, List, Optional +import warnings + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + __test__ = False + + def __init__(self, *responses: Response): + self.index = 0 + self.responses = responses + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = self.responses[self.index] + self.index += 1 + return response + + +def get_api(*responses: Response) -> WebApi: + return WebApi(Session("", "", test_auth2_session=TestAuth2Session(*responses))) + + +def get_json_response( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", + entities: Optional[List[Dict[str, str]]] = None, +) -> Response: + if entities is None: + entities = [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ] + json = json_dumps( + { + "downloadFullListOnOrAfter": downloadFullListOnOrAfter, + "timeStampForIfModifiedSince": timeStampForIfModifiedSince, + "state": state, + "entities": entities, + } + ) + response = Response() + response.status_code = 200 + response.raw = BytesIO(bytes(json, "utf-8")) + return response + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + +def test_full_listing_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + raise Exception("Test exception") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(3) + assert is_aborted is False + assert exception is not None + self.abort() + + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) + + _TestDataPackageListPoller(api).start() + + assert hit == 3 + + +def test_full_listing_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + + def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(3) + raise Exception("Test exception") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(4) + assert is_aborted is False + assert exception is not None + self.abort() + + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) + + _TestDataPackageListPoller(api).start() + + assert hit == 4 + + +def test_full_listing_3() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + + def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(3) + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(4) + self.abort() + raise Exception("Test exception") + + def sleep(self, secs: float) -> None: + hit_test(5) + assert secs == self.on_error_delay + + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) + + with warnings.catch_warnings(record=True): + _TestDataPackageListPoller(api).start() + + assert hit == 5 + + +# test_abort_listing + + +def test_listing_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + raise Exception("Test exception") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(4) + self.abort() + + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 4 + + +def test_listing_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(4) + raise Exception("Test exception") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + assert is_aborted is False + assert exception is not None + self.abort() + + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 5 + + +def test_listing_3() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + hit_test(4) + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + self.abort() + raise Exception("Test exception") + + def sleep(self, secs: float) -> None: + hit_test(6) + assert secs == self.on_error_delay + + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) + + with warnings.catch_warnings(record=True): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 6 + + +# test_abort_listing_and_listing_incomplete + + +def test_listing_and_listing_incomplete_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(5) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + if hit_test(4, 6) == 6: + raise Exception("Test exception") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is False + assert exception is not None + self.abort() + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 7 + + +def test_listing_and_listing_incomplete_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + if hit_test(5, 8) == 8: + assert secs == self.on_error_delay + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4, 6) + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is False + assert exception is None + self.abort() + raise Exception("Test exception") + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) + ) + + with warnings.catch_warnings(record=True): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 8 diff --git a/tests/Web/data_package_list_poller/normal.py b/tests/Web/data_package_list_poller/normal.py new file mode 100644 index 00000000..fe046b05 --- /dev/null +++ b/tests/Web/data_package_list_poller/normal.py @@ -0,0 +1,292 @@ +from datetime import datetime, timezone +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, Dict, List, Optional + +import pytest + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + __test__ = False + + def __init__(self, *responses: Response): + self.index = 0 + self.responses = responses + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = self.responses[self.index] + self.index += 1 + return response + + +def get_api(*responses: Response) -> WebApi: + return WebApi(Session("", "", test_auth2_session=TestAuth2Session(*responses))) + + +def get_json_response( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", + entities: Optional[List[Dict[str, str]]] = None, +) -> Response: + if entities is None: + entities = [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ] + json = json_dumps( + { + "downloadFullListOnOrAfter": downloadFullListOnOrAfter, + "timeStampForIfModifiedSince": timeStampForIfModifiedSince, + "state": state, + "entities": entities, + } + ) + response = Response() + response.status_code = 200 + response.raw = BytesIO(bytes(json, "utf-8")) + return response + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + +# _run_full_listing +def test_full_listing() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(7) + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(2) + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.FULL_LISTING + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.FULL_LISTING + if hit == 3: + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + if hit == 4: + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + if hit == 5: + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(6) + assert is_aborted is False + assert exception is None + + api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller(api, chunk_size=1).start() + + assert hit == 7 + + +# _run_listing +def test_listing() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(8) + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.UP_TO_DATE + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + assert subscription.state == DataPackageListState.UP_TO_DATE + if hit == 4: + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 5: + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 6: + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + else: + raise Exception("should not be here") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(7) + assert is_aborted is False + assert exception is None + + api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + chunk_size=1, + ).start() + + assert hit == 8 + + +# _run_listing and _run_listing_incomplete +def test_listing_and_listing_incomplete() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(7, 12) + if hit == 7: + assert secs == self.incomplete_delay + elif hit == 12: + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + if hit_test(3) == 3: + assert subscription.state == DataPackageListState.INCOMPLETE + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + nonlocal hit + hit += 1 + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + if hit == 4: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 5: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 6: + assert subscription.state == DataPackageListState.INCOMPLETE + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + elif hit == 8: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6))] + elif hit == 9: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6))] + elif hit == 10: + assert subscription.state == DataPackageListState.UP_TO_DATE + assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] + else: + raise Exception("should not be here") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(11) + assert is_aborted is False + assert exception is None + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) + ) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + chunk_size=1, + ).start() + + assert hit == 12 diff --git a/tests/Web/data_package_list_poller/retry.py b/tests/Web/data_package_list_poller/retry.py new file mode 100644 index 00000000..1b4e878b --- /dev/null +++ b/tests/Web/data_package_list_poller/retry.py @@ -0,0 +1,239 @@ +from datetime import datetime, timezone +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, Dict, List, Optional + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + __test__ = False + + def __init__(self, *responses: Response): + self.index = 0 + self.responses = responses + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = self.responses[self.index] + self.index += 1 + return response + + +def get_api(*responses: Response) -> WebApi: + return WebApi(Session("", "", test_auth2_session=TestAuth2Session(*responses))) + + +def get_json_response( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", + entities: Optional[List[Dict[str, str]]] = None, +) -> Response: + if entities is None: + entities = [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ] + json = json_dumps( + { + "downloadFullListOnOrAfter": downloadFullListOnOrAfter, + "timeStampForIfModifiedSince": timeStampForIfModifiedSince, + "state": state, + "entities": entities, + } + ) + response = Response() + response.status_code = 200 + response.raw = BytesIO(bytes(json, "utf-8")) + return response + + +def http_500_response() -> Response: + response = Response() + response.status_code = 500 + return response + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + +# _run_full_listing +def test_full_listing() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(2) + assert secs == self.on_error_delay + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4) + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(5) + self.abort() + + api = get_api(http_500_response(), get_json_response(DataPackageListState.FULL_LISTING)) + + _TestDataPackageListPoller(api, chunk_size=200).start() + + assert hit == 5 + + +# _run_listing +def test_listing() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(3) + assert secs == self.on_error_delay + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(4) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(5) + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(6) + self.abort() + + api = get_api(http_500_response(), get_json_response(DataPackageListState.UP_TO_DATE)) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 6 + + +# _run_listing and _run_listing_incomplete +def test_listing_and_listing_incomplete() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(5, 6, 12) + if hit == 5: + assert secs == self.incomplete_delay + elif hit == 6: + assert secs == self.on_error_delay + elif hit == 12: + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + if hit_test(4, 7) == 4: + assert subscription.state == DataPackageListState.INCOMPLETE + elif hit == 7: + assert subscription.state == DataPackageListState.UP_TO_DATE + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(8) + self.abort() + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), + http_500_response(), + get_json_response(DataPackageListState.UP_TO_DATE), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 8 diff --git a/tests/Web/data_package_list_poller/test_access.py b/tests/Web/data_package_list_poller/test_access.py new file mode 100644 index 00000000..97b85bbb --- /dev/null +++ b/tests/Web/data_package_list_poller/test_access.py @@ -0,0 +1,91 @@ +from datetime import datetime +from typing import Any, List, Optional + +import pytest + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem + + +class TestAuth2Session: + __test__ = False + + def __init__(self, *responses: Response): + self.index = 0 + self.responses = responses + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = self.responses[self.index] + self.index += 1 + return response + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + def on_incremental_start(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + raise Exception("should not be called") + + +def test_access() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestAuth2Session: + __test__ = False + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + hit_test(1) + response = Response() + response.status_code = 403 + return response + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + api = WebApi(Session("", "", test_auth2_session=_TestAuth2Session())) + with pytest.raises(Exception, match="Needs access - The account is not set up to use DataPackageList"): + _TestDataPackageListPoller(api).start() + + assert hit == 1 From 10529a2c66f095a1fcafd43e09bc34e0621c2664 Mon Sep 17 00:00:00 2001 From: mb-jp Date: Tue, 24 Oct 2023 13:15:20 +0200 Subject: [PATCH 10/12] wip --- .../web/data_package_list_poller.py | 101 +++++------ tests/Web/data_package_list_poller/abort.py | 56 +++--- tests/Web/data_package_list_poller/error.py | 153 +++++++---------- tests/Web/data_package_list_poller/normal.py | 30 ++-- tests/Web/data_package_list_poller/retry.py | 160 ++++++++++++++++-- .../data_package_list_poller/test_access.py | 12 +- 6 files changed, 297 insertions(+), 215 deletions(-) diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index fcdd13bf..da78c941 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -3,7 +3,6 @@ import time from typing import List, Optional -import warnings from .web_api import WebApi from .web_types.data_package_list_state import DataPackageListState @@ -12,6 +11,10 @@ from .web_types.data_package_body import DataPackageBody +class RrytryException(Exception): + ... + + class DataPackageListPoller(ABC): """ This is work in progress and might change soon. @@ -49,6 +52,7 @@ def __init__( """ The time to wait, in seconds, between continuing partial updates. """ self.on_error_delay = 30 """ The time to wait, in seconds, before retrying after an error. """ + self.on_retry_delay = 30 self._sleep = time.sleep self._now = lambda: datetime.now(timezone.utc) @@ -108,10 +112,9 @@ def _test_access(self) -> None: raise Exception("Needs access - The account is not set up to use DataPackageList") def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: - is_running = False - attempt = 0 + is_stated = False + attempt = 1 while True: - attempt += 1 try: with self._api.get_data_package_list_chunked(None, self._chunk_size) as context: body = DataPackageBody( @@ -119,42 +122,34 @@ def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: context.download_full_list_on_or_after, context.state, ) - is_running = True - self.on_full_listing_start(body) + is_stated = True + self.on_full_listing_begin(body) if self._abort: - self._run_on_full_listing_stop(True, None) + self.on_full_listing_end(True, None) return None for items in context.items: - self.on_full_listing_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + self.on_full_listing_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) if self._abort: - self._run_on_full_listing_stop(True, None) + self.on_full_listing_end(True, None) return None - self._run_on_full_listing_stop(False, None) + self.on_full_listing_end(False, None) return body except Exception as ex: # pylint: disable=broad-except - if is_running: - self._run_on_full_listing_stop(False, ex) - return None - if attempt > max_attempts: + if is_stated: raise - self._sleep(self.on_error_delay) - - def _run_on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - try: - self.on_full_listing_stop(is_aborted, exception) - except Exception as ex: # pylint: disable=broad-except - warnings.warn("Unhandled exception in on_full_listing_stop, " + str(ex)) - self._sleep(self.on_error_delay) + if attempt > max_attempts: + raise RrytryException("Rrytry Exception") from ex + self._sleep(self.on_retry_delay * attempt) + attempt += 1 def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: is_stated = False - attempt = 0 + attempt = 1 while True: - attempt += 1 try: with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: body = DataPackageBody( @@ -163,34 +158,33 @@ def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Op context.state, ) is_stated = True - self.on_incremental_start(body) + self.on_incremental_begin(body) if self._abort: - self._run_on_incremental_stop(True, None) + self.on_incremental_end(True, None) return None for items in context.items: - self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + self.on_incremental_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) if self._abort: - self._run_on_incremental_stop(True, None) + self.on_incremental_end(True, None) return None if body.state == DataPackageListState.UP_TO_DATE: - self._run_on_incremental_stop(False, None) + self.on_incremental_end(False, None) return body except Exception as ex: # pylint: disable=broad-except if is_stated: - self._run_on_incremental_stop(False, ex) - return None - if attempt > max_attempts: raise - self._sleep(self.on_error_delay) + if attempt > max_attempts: + raise RrytryException("Rrytry Exception") from ex + self._sleep(self.on_retry_delay * attempt) + attempt += 1 def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: is_stated = False - attempt = 0 + attempt = 1 while True: - attempt += 1 try: with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: body = DataPackageBody( @@ -200,13 +194,13 @@ def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int ) is_stated = True for items in context.items: - self.on_incremental_items(body, [DataPackageListItem(x[0], x[1]) for x in items]) + self.on_incremental_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) if self._abort: - self._run_on_incremental_stop(True, None) + self.on_incremental_end(True, None) return None if body.state == DataPackageListState.UP_TO_DATE: - self._run_on_incremental_stop(False, None) + self.on_incremental_end(False, None) return body self._sleep(self.incomplete_delay) @@ -214,31 +208,28 @@ def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int if_modified_since = body.time_stamp_for_if_modified_since except Exception as ex: # pylint: disable=broad-except if is_stated: - self._run_on_incremental_stop(False, ex) - return None - if attempt > max_attempts: raise - self._sleep(self.on_error_delay) - - def _run_on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - try: - self.on_incremental_stop(is_aborted, exception) - except Exception as ex: # pylint: disable=broad-except - warnings.warn("Unhandled exception in on_incremental_stop, " + str(ex)) - self._sleep(self.on_error_delay) + if attempt > max_attempts: + try: + raise RrytryException("Rrytry Exception") from ex + except Exception as retry_ex: # pylint: disable=broad-except + self.on_incremental_end(False, retry_ex) + return None + self._sleep(self.on_retry_delay * attempt) + attempt += 1 # full_listing @abstractmethod - def on_full_listing_start(self, subscription: DataPackageBody) -> None: + def on_full_listing_begin(self, subscription: DataPackageBody) -> None: """This override is called when a full listing starts.""" @abstractmethod - def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: """This override is called repeatedly with one or more items until all items are listed.""" @abstractmethod - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: """ This override is called when the full listing is stopped. Parameters @@ -252,15 +243,15 @@ def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) # listing @abstractmethod - def on_incremental_start(self, subscription: DataPackageBody) -> None: + def on_incremental_begin(self, subscription: DataPackageBody) -> None: """This override is called when an incremental listing starts.""" @abstractmethod - def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: """This override is called repeatedly with one or more items until all updated items are listed.""" @abstractmethod - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: """ This override is called when the incremental listing is stopped. Parameters diff --git a/tests/Web/data_package_list_poller/abort.py b/tests/Web/data_package_list_poller/abort.py index 5ccf65e8..893a6a76 100644 --- a/tests/Web/data_package_list_poller/abort.py +++ b/tests/Web/data_package_list_poller/abort.py @@ -74,22 +74,22 @@ def sleep(self, secs: float) -> None: def now(self) -> datetime: raise Exception("should not be called") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") @@ -108,11 +108,11 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) self.abort() - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(3) assert is_aborted is True assert exception is None @@ -139,14 +139,14 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) - def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: if hit_test(3, 4) == 4: self.abort() - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) assert is_aborted is True assert exception is None @@ -173,13 +173,13 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) - def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(3) - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(4) assert is_aborted is False assert exception is None @@ -214,11 +214,11 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) self.abort() - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(4) assert is_aborted is True assert exception is None @@ -253,14 +253,14 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: if hit_test(4): self.abort() - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) assert is_aborted is True assert exception is None @@ -296,13 +296,13 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(4) - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) assert is_aborted is False assert exception is None @@ -344,14 +344,14 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: if hit_test(4, 6) == 6: self.abort() - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(7) assert is_aborted is True assert exception is None @@ -391,13 +391,13 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4, 6) - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(7) assert is_aborted is False assert exception is None diff --git a/tests/Web/data_package_list_poller/error.py b/tests/Web/data_package_list_poller/error.py index ff237b99..1596e501 100644 --- a/tests/Web/data_package_list_poller/error.py +++ b/tests/Web/data_package_list_poller/error.py @@ -2,7 +2,7 @@ from io import BytesIO from json import dumps as json_dumps from typing import Any, Dict, List, Optional -import warnings +import pytest from requests import Response @@ -75,22 +75,22 @@ def sleep(self, secs: float) -> None: def now(self) -> datetime: raise Exception("should not be called") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") @@ -109,21 +109,16 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) raise Exception("Test exception") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(3) - assert is_aborted is False - assert exception is not None - self.abort() - api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) - _TestDataPackageListPoller(api).start() + with pytest.raises(Exception, match="Test exception"): + _TestDataPackageListPoller(api).start() - assert hit == 3 + assert hit == 2 def test_full_listing_2() -> None: @@ -141,24 +136,19 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) - def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(3) raise Exception("Test exception") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(4) - assert is_aborted is False - assert exception is not None - self.abort() - api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) - _TestDataPackageListPoller(api).start() + with pytest.raises(Exception, match="Test exception"): + _TestDataPackageListPoller(api).start() - assert hit == 4 + assert hit == 3 def test_full_listing_3() -> None: @@ -176,27 +166,22 @@ class _TestDataPackageListPoller(TestDataPackageListPoller): def _test_access(self) -> None: hit_test(1) - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) - def on_full_listing_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(3) - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(4) - self.abort() raise Exception("Test exception") - def sleep(self, secs: float) -> None: - hit_test(5) - assert secs == self.on_error_delay - api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) - with warnings.catch_warnings(record=True): + with pytest.raises(Exception, match="Test exception"): _TestDataPackageListPoller(api).start() - assert hit == 5 + assert hit == 4 # test_abort_listing @@ -221,23 +206,20 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) raise Exception("Test exception") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(4) - self.abort() - api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - ).start() + with pytest.raises(Exception, match="Test exception"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() - assert hit == 4 + assert hit == 3 def test_listing_2() -> None: @@ -259,28 +241,23 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(4) raise Exception("Test exception") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(5) - assert is_aborted is False - assert exception is not None - self.abort() - api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - ).start() + with pytest.raises(Exception, match="Test exception"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() - assert hit == 5 + assert hit == 4 def test_listing_3() -> None: @@ -302,31 +279,26 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: + def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(4) - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) - self.abort() raise Exception("Test exception") - def sleep(self, secs: float) -> None: - hit_test(6) - assert secs == self.on_error_delay - api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) - with warnings.catch_warnings(record=True): + with pytest.raises(Exception, match="Test exception"): _TestDataPackageListPoller( api, download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), ).start() - assert hit == 6 + assert hit == 5 # test_abort_listing_and_listing_incomplete @@ -354,30 +326,25 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: if hit_test(4, 6) == 6: raise Exception("Test exception") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(7) - assert is_aborted is False - assert exception is not None - self.abort() - api = get_api( get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) ) - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - ).start() + with pytest.raises(Exception, match="Test exception"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() - assert hit == 7 + assert hit == 6 def test_listing_and_listing_incomplete_2() -> None: @@ -396,35 +363,31 @@ def _test_access(self) -> None: hit_test(1) def sleep(self, secs: float) -> None: - if hit_test(5, 8) == 8: - assert secs == self.on_error_delay + hit_test(5) def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4, 6) - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(7) - assert is_aborted is False - assert exception is None - self.abort() raise Exception("Test exception") api = get_api( get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) ) - with warnings.catch_warnings(record=True): + with pytest.raises(Exception, match="Test exception"): _TestDataPackageListPoller( api, download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), ).start() - assert hit == 8 + assert hit == 7 diff --git a/tests/Web/data_package_list_poller/normal.py b/tests/Web/data_package_list_poller/normal.py index fe046b05..5b8de3db 100644 --- a/tests/Web/data_package_list_poller/normal.py +++ b/tests/Web/data_package_list_poller/normal.py @@ -76,22 +76,22 @@ def sleep(self, secs: float) -> None: def now(self) -> datetime: raise Exception("should not be called") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") @@ -116,13 +116,13 @@ def sleep(self, secs: float) -> None: assert secs == self.up_to_date_delay raise Exception("End of test") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) assert subscription.state == DataPackageListState.FULL_LISTING - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: nonlocal hit hit += 1 assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) @@ -135,7 +135,7 @@ def on_full_listing_items(self, subscription: "DataPackageBody", items: List["Da if hit == 5: assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(6) assert is_aborted is False assert exception is None @@ -173,13 +173,13 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) assert subscription.state == DataPackageListState.UP_TO_DATE - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: nonlocal hit hit += 1 assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) @@ -194,7 +194,7 @@ def on_incremental_items(self, subscription: "DataPackageBody", items: List["Dat else: raise Exception("should not be here") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(7) assert is_aborted is False assert exception is None @@ -240,13 +240,13 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) if hit_test(3) == 3: assert subscription.state == DataPackageListState.INCOMPLETE - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: nonlocal hit hit += 1 assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) @@ -272,7 +272,7 @@ def on_incremental_items(self, subscription: "DataPackageBody", items: List["Dat else: raise Exception("should not be here") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(11) assert is_aborted is False assert exception is None diff --git a/tests/Web/data_package_list_poller/retry.py b/tests/Web/data_package_list_poller/retry.py index 1b4e878b..483938e6 100644 --- a/tests/Web/data_package_list_poller/retry.py +++ b/tests/Web/data_package_list_poller/retry.py @@ -2,11 +2,12 @@ from io import BytesIO from json import dumps as json_dumps from typing import Any, Dict, List, Optional +import pytest from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, RrytryException from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -80,22 +81,22 @@ def sleep(self, secs: float) -> None: def now(self) -> datetime: raise Exception("should not be called") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") @@ -119,13 +120,13 @@ def sleep(self, secs: float) -> None: hit_test(2) assert secs == self.on_error_delay - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4) - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(5) self.abort() @@ -136,6 +137,37 @@ def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) assert hit == 5 +def test_full_listing_error() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(2, 3, 4) - 1 + assert secs == self.on_retry_delay * hit + + api = get_api( + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + with pytest.raises(RrytryException, match="Rrytry Exception"): + _TestDataPackageListPoller(api, chunk_size=200).start() + + assert hit == 4 + + # _run_listing def test_listing() -> None: hit = 0 @@ -160,13 +192,13 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(4) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(5) - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(6) self.abort() @@ -181,6 +213,46 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) assert hit == 6 +def test_listing_error() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def sleep(self, secs: float) -> None: + hit = hit_test(3, 4, 5) - 2 + assert secs == self.on_retry_delay * hit + + api = get_api( + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + + with pytest.raises(RrytryException, match="Rrytry Exception"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 5 + + # _run_listing and _run_listing_incomplete def test_listing_and_listing_incomplete() -> None: hit = 0 @@ -211,16 +283,16 @@ def now(self) -> datetime: hit_test(2) return datetime(2000, 1, 1, tzinfo=timezone.utc) - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: if hit_test(4, 7) == 4: assert subscription.state == DataPackageListState.INCOMPLETE elif hit == 7: assert subscription.state == DataPackageListState.UP_TO_DATE - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: hit_test(8) self.abort() @@ -237,3 +309,59 @@ def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) ).start() assert hit == 8 + + +def test_listing_and_listing_incomplete_error() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(5, 6, 7, 8) + if hit == 5: + assert secs == self.incomplete_delay + else: + assert secs == self.on_retry_delay * (hit - 5) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4) + assert subscription.state == DataPackageListState.INCOMPLETE + + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + hit_test(9) + self.abort() + assert is_aborted is False + assert exception is not None + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 9 diff --git a/tests/Web/data_package_list_poller/test_access.py b/tests/Web/data_package_list_poller/test_access.py index 97b85bbb..e8bf5b21 100644 --- a/tests/Web/data_package_list_poller/test_access.py +++ b/tests/Web/data_package_list_poller/test_access.py @@ -44,22 +44,22 @@ def sleep(self, secs: float) -> None: def now(self) -> datetime: raise Exception("should not be called") - def on_full_listing_start(self, subscription: "DataPackageBody") -> None: + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_full_listing_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") - def on_incremental_start(self, subscription: "DataPackageBody") -> None: + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: raise Exception("should not be called") - def on_incremental_items(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_stop(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: raise Exception("should not be called") From 0a3130099a78f478a3cef37b030b77e00c919121 Mon Sep 17 00:00:00 2001 From: Thomas Olsson <72909585+mb-to@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:29:41 +0200 Subject: [PATCH 11/12] Improved documentation and some spelling --- macrobond_data_api/web/_web_only_api.py | 16 ++++++++------ .../web/data_package_list_poller.py | 22 +++++++++---------- tests/Web/data_package_list_poller/retry.py | 6 ++--- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/macrobond_data_api/web/_web_only_api.py b/macrobond_data_api/web/_web_only_api.py index db502c85..1e35fd55 100644 --- a/macrobond_data_api/web/_web_only_api.py +++ b/macrobond_data_api/web/_web_only_api.py @@ -144,11 +144,11 @@ def get_data_package_list_iterative( items_callback : Callable[[macrobond_data_api.web.web_types.data_package_body.DataPackageBody, List[macrobond_data_api.web.web_types.data_pacakge_list_item.DataPackageListItem]], Optional[bool]] The callback for each batch of items. Return True to continue processing. - if_modified_since : datetime + if_modified_since : datetime, optional The timestamp of the property time_stamp_for_if_modified_since from the response of the previous call. If not specified, all items will be returned. - buffer_size : int + buffer_size : int, optional The maximum number of items to include in each callback Returns ------- @@ -199,18 +199,19 @@ def get_data_package_list_chunked( # pylint: disable=line-too-long """ Process the data package list in chunks. - This is more efficient since the complete list does not have to be in memory. + This is more efficient since the complete list does not have to be in memory and it can be processed while + downloading. Typically you want to pass the date of time_stamp_for_if_modified_since from response of the previous call to get incremental updates. Parameters ---------- - if_modified_since : datetime + if_modified_since : datetime, optional The timestamp of the property time_stamp_for_if_modified_since from the response of the previous call. If not specified, all items will be returned. - chunk_size : int + chunk_size : int, optional The maximum number of items to include in each List in DataPackageListContext.items Returns ------- @@ -226,6 +227,7 @@ def get_data_package_list_chunked( def entity_search_multi_filter_long( self: "WebApi", *filters: "SearchFilter", include_discontinued: bool = False ) -> SearchResultLong: + # pylint: disable=line-too-long """ Search for time series and other entitites. This call can return more results than `macrobond_data_api.common.api.Api.entity_search_multi_filter`, @@ -238,12 +240,12 @@ def entity_search_multi_filter_long( ---------- *filters : `macrobond_data_api.common.types.search_filter.SearchFilter` One or more search filters. - include_discontinued : bool + include_discontinued : bool, optional Set this value to True in order to include discontinued entities in the search. Returns ------- - `macrobond_data_api.common.types.search_result_long.SearchResultLong` + A `macrobond_data_api.common.types.search_result_long.SearchResultLong` object containing the names of the entities that match the search filters. """ def convert_filter_to_web_filter(_filter: "SearchFilter") -> "WebSearchFilter": diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index da78c941..6f7e72e6 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -11,7 +11,7 @@ from .web_types.data_package_body import DataPackageBody -class RrytryException(Exception): +class RetryException(Exception): ... @@ -19,19 +19,19 @@ class DataPackageListPoller(ABC): """ This is work in progress and might change soon. Run a loop polling for changed series in the data package list. - Derive from this class and override `on_full_listing_start`, `on_full_listing_items`, `on_full_listing_stop`, - `on_incremental_start`, `on_incremental_items` and `on_incremental_stop`. + Derive from this class and override `on_full_listing_begin`, `on_full_listing_batch`, `on_full_listing_end`, + `on_incremental_begin`, `on_incremental_batch` and `on_incremental_end`. Parameters ---------- api : WebApi The API instance to use. - download_full_list_on_or_after : datetime + download_full_list_on_or_after : datetime, optional The saved value of `download_full_list_on_or_after` from the previous run. `None` on first run. - time_stamp_for_if_modified_since: datetime + time_stamp_for_if_modified_since: datetime, optional The saved value of `time_stamp_for_if_modified_since` from the previous run. `None`on first run. - chunk_size : int - The maximum number of items to include in each on_*_items() + chunk_size : int, optional + The maximum number of items to include in each on_*_batch() """ def __init__( @@ -73,7 +73,7 @@ def download_full_list_on_or_after(self) -> Optional[datetime]: @property def time_stamp_for_if_modified_since(self) -> Optional[datetime]: """ - This value is used internall to keep track of the the time of the last detected modification. + This value is used internally to keep track of the the time of the last detected modification. Save this value after processing and pass in constructor for the next run. """ return self._time_stamp_for_if_modified_since @@ -142,7 +142,7 @@ def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: if is_stated: raise if attempt > max_attempts: - raise RrytryException("Rrytry Exception") from ex + raise RetryException("Retry Exception") from ex self._sleep(self.on_retry_delay * attempt) attempt += 1 @@ -177,7 +177,7 @@ def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Op if is_stated: raise if attempt > max_attempts: - raise RrytryException("Rrytry Exception") from ex + raise RetryException("Retry Exception") from ex self._sleep(self.on_retry_delay * attempt) attempt += 1 @@ -211,7 +211,7 @@ def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int raise if attempt > max_attempts: try: - raise RrytryException("Rrytry Exception") from ex + raise RetryException("Retry Exception") from ex except Exception as retry_ex: # pylint: disable=broad-except self.on_incremental_end(False, retry_ex) return None diff --git a/tests/Web/data_package_list_poller/retry.py b/tests/Web/data_package_list_poller/retry.py index 483938e6..36daf3ce 100644 --- a/tests/Web/data_package_list_poller/retry.py +++ b/tests/Web/data_package_list_poller/retry.py @@ -7,7 +7,7 @@ from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, RrytryException +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, RetryException from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -162,7 +162,7 @@ def sleep(self, secs: float) -> None: http_500_response(), http_500_response(), ) - with pytest.raises(RrytryException, match="Rrytry Exception"): + with pytest.raises(RetryException, match="Retry Exception"): _TestDataPackageListPoller(api, chunk_size=200).start() assert hit == 4 @@ -243,7 +243,7 @@ def sleep(self, secs: float) -> None: http_500_response(), ) - with pytest.raises(RrytryException, match="Rrytry Exception"): + with pytest.raises(RetryException, match="Retry Exception"): _TestDataPackageListPoller( api, download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), From f4cf814bc748926ed07f10915df85a8a57595b83 Mon Sep 17 00:00:00 2001 From: mb-jp Date: Thu, 26 Oct 2023 11:57:23 +0200 Subject: [PATCH 12/12] wip --- .../web/data_package_list_poller.py | 279 ++++++++----- .../web_types/data_package_list_context.py | 3 +- tests/Web/data_package_list_poller/abort.py | 33 +- tests/Web/data_package_list_poller/error.py | 15 +- tests/Web/data_package_list_poller/normal.py | 91 ++++- .../data_package_list_poller/on_exception.py | 378 ++++++++++++++++++ tests/Web/data_package_list_poller/retry.py | 160 +------- .../data_package_list_poller/test_access.py | 9 +- 8 files changed, 689 insertions(+), 279 deletions(-) create mode 100644 tests/Web/data_package_list_poller/on_exception.py diff --git a/macrobond_data_api/web/data_package_list_poller.py b/macrobond_data_api/web/data_package_list_poller.py index 6f7e72e6..8f648b6f 100644 --- a/macrobond_data_api/web/data_package_list_poller.py +++ b/macrobond_data_api/web/data_package_list_poller.py @@ -1,8 +1,10 @@ from abc import ABC, abstractmethod +from enum import Enum from datetime import datetime, timezone import time -from typing import List, Optional +from typing import List, Optional, Tuple, Union, Callable, TYPE_CHECKING + from .web_api import WebApi from .web_types.data_package_list_state import DataPackageListState @@ -10,9 +12,57 @@ from .web_types.data_pacakge_list_item import DataPackageListItem from .web_types.data_package_body import DataPackageBody +if TYPE_CHECKING: + from macrobond_data_api.web.web_types.data_package_list_context import ( + DataPackageListContext, + DataPackageListContextManager, + ) + + +class ExceptionSource(Enum): + FAILED_TO_BEGIN_FULL_LISTING = 1 + """ + Failed_begin_full_listing can happen after: + * start() + * on_exception() + * on_full_listing_end() + * on_incremental_end() + """ + + FAILED_TO_BEGIN_LISTING = 2 + """ + Failed_begin_listing + * start() + * on_exception() + * on_full_listing_end() + * on_incremental_end() + """ + + FAILED_TO_BEGIN_LISTING_INCOMPLETE = 3 + """ + Failed_to_begin_listing_incomplete + * on_incremental_batch() + """ -class RetryException(Exception): - ... + FAILED_TO_GET_BATCH_IN_FULL_LISTING = 4 + """ + Failed_to_get_batch_in_full_listing can happen after: + * on_full_listing_begin() + * on_full_listing_batch() + """ + + FAILED_TO_GET_BATCH_IN_LISTING = 5 + """ + Failed_to_get_batch_in_listing can happen after: + * on_incremental_begin() + * on_incremental_batch() + """ + + FAILED_TO_GET_BATCH_IN_LISTING_INCOMPLETE = 6 + """ + Failed_to_get_batch_in_listing_incomplete can happen after: + * on_incremental_batch() + """ class DataPackageListPoller(ABC): @@ -112,112 +162,147 @@ def _test_access(self) -> None: raise Exception("Needs access - The account is not set up to use DataPackageList") def _run_full_listing(self, max_attempts: int = 3) -> Optional[DataPackageBody]: - is_stated = False - attempt = 1 - while True: - try: - with self._api.get_data_package_list_chunked(None, self._chunk_size) as context: - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, - ) - is_stated = True - self.on_full_listing_begin(body) - - if self._abort: - self.on_full_listing_end(True, None) - return None - - for items in context.items: - self.on_full_listing_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) - if self._abort: - self.on_full_listing_end(True, None) - return None - - self.on_full_listing_end(False, None) - - return body - except Exception as ex: # pylint: disable=broad-except - if is_stated: - raise - if attempt > max_attempts: - raise RetryException("Retry Exception") from ex - self._sleep(self.on_retry_delay * attempt) - attempt += 1 + context_manager: Optional["DataPackageListContextManager"] = None + try: + context_manager, context, body = self._retry_get_data_package_list_chunked( + max_attempts, None, ExceptionSource.FAILED_TO_BEGIN_FULL_LISTING + ) + if context is None or body is None: + return None + + self.on_full_listing_begin(body) + if self._abort: + self.on_full_listing_end(True) + return None + + if self._try_iterator( + context, + body, + self.on_full_listing_batch, + self.on_full_listing_end, + ExceptionSource.FAILED_TO_GET_BATCH_IN_FULL_LISTING, + ): + self.on_full_listing_end(False) + return body + return None + finally: + if context_manager: + context_manager.__exit__(None, None, None) def _run_listing(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: - is_stated = False - attempt = 1 - while True: - try: - with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, - ) - is_stated = True - self.on_incremental_begin(body) - - if self._abort: - self.on_incremental_end(True, None) - return None + context_manager: Optional["DataPackageListContextManager"] = None + try: + context_manager, context, body = self._retry_get_data_package_list_chunked( + max_attempts, if_modified_since, ExceptionSource.FAILED_TO_BEGIN_LISTING + ) + if context is None or body is None: + return None + + self.on_incremental_begin(body) + if self._abort: + self.on_incremental_end(True) + return None + + if ( + self._try_iterator( + context, + body, + self.on_incremental_batch, + self.on_incremental_end, + ExceptionSource.FAILED_TO_GET_BATCH_IN_LISTING, + ) + is False + ): + return None - for items in context.items: - self.on_incremental_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) - if self._abort: - self.on_incremental_end(True, None) - return None + if body.state == DataPackageListState.UP_TO_DATE: + self.on_incremental_end(False) - if body.state == DataPackageListState.UP_TO_DATE: - self.on_incremental_end(False, None) - return body - except Exception as ex: # pylint: disable=broad-except - if is_stated: - raise - if attempt > max_attempts: - raise RetryException("Retry Exception") from ex - self._sleep(self.on_retry_delay * attempt) - attempt += 1 + return body + finally: + if context_manager: + context_manager.__exit__(None, None, None) def _run_listing_incomplete(self, if_modified_since: datetime, max_attempts: int = 3) -> Optional[DataPackageBody]: - is_stated = False - attempt = 1 while True: + context_manager: Optional["DataPackageListContextManager"] = None try: - with self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) as context: - body = DataPackageBody( - context.time_stamp_for_if_modified_since, - context.download_full_list_on_or_after, - context.state, + context_manager, context, body = self._retry_get_data_package_list_chunked( + max_attempts, if_modified_since, ExceptionSource.FAILED_TO_BEGIN_LISTING_INCOMPLETE + ) + if context is None or body is None: + return None + + if ( + self._try_iterator( + context, + body, + self.on_incremental_batch, + self.on_incremental_end, + ExceptionSource.FAILED_TO_GET_BATCH_IN_LISTING_INCOMPLETE, ) - is_stated = True - for items in context.items: - self.on_incremental_batch(body, [DataPackageListItem(x[0], x[1]) for x in items]) - if self._abort: - self.on_incremental_end(True, None) - return None + is False + ): + return None if body.state == DataPackageListState.UP_TO_DATE: - self.on_incremental_end(False, None) + self.on_incremental_end(False) return body self._sleep(self.incomplete_delay) if_modified_since = body.time_stamp_for_if_modified_since + finally: + if context_manager: + context_manager.__exit__(None, None, None) + + def _retry_get_data_package_list_chunked( + self, max_attempts: int, if_modified_since: Optional[datetime], exception_source: ExceptionSource + ) -> Union[ + Tuple["DataPackageListContextManager", "DataPackageListContext", DataPackageBody], + Tuple["DataPackageListContextManager", None, None], + ]: + attempt = 1 + while True: + try: + context_manager = self._api.get_data_package_list_chunked(if_modified_since, self._chunk_size) + context = context_manager.__enter__() # pylint: disable=unnecessary-dunder-call + body = DataPackageBody( + context.time_stamp_for_if_modified_since, + context.download_full_list_on_or_after, + context.state, + ) + return context_manager, context, body except Exception as ex: # pylint: disable=broad-except - if is_stated: - raise if attempt > max_attempts: - try: - raise RetryException("Retry Exception") from ex - except Exception as retry_ex: # pylint: disable=broad-except - self.on_incremental_end(False, retry_ex) - return None + self.on_exception(exception_source, ex) + return context_manager, None, None self._sleep(self.on_retry_delay * attempt) attempt += 1 + def _try_iterator( + self, + context: "DataPackageListContext", + body: DataPackageBody, + on_batch: Callable[[DataPackageBody, List[DataPackageListItem]], None], + on_end: Callable[[bool], None], + exception_source: ExceptionSource, + ) -> bool: + iterator = iter(context.items) + while True: + try: + items = [DataPackageListItem(x[0], x[1]) for x in next(iterator)] + except StopIteration: + return True + except Exception as ex: # pylint: disable=broad-except + self.on_exception(exception_source, ex) + return False + + on_batch(body, items) + if self._abort: + on_end(True) + return False + # full_listing @abstractmethod @@ -229,15 +314,13 @@ def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataP """This override is called repeatedly with one or more items until all items are listed.""" @abstractmethod - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: """ This override is called when the full listing is stopped. Parameters ---------- is_aborted : bool The processing was aborted. - exception : Optional[Exception] - If not None, there was an exception. """ # listing @@ -251,15 +334,23 @@ def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPa """This override is called repeatedly with one or more items until all updated items are listed.""" @abstractmethod - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: """ This override is called when the incremental listing is stopped. Parameters ---------- is_aborted : bool The processing was aborted. - exception : Optional[Exception] - If not None, there was an exception. + """ + + @abstractmethod + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + """ + This override is called when the incremental listing is stopped. + Parameters + ---------- + exception : Exception + The exception. """ def abort(self) -> None: diff --git a/macrobond_data_api/web/web_types/data_package_list_context.py b/macrobond_data_api/web/web_types/data_package_list_context.py index e5cb042f..5015ba31 100644 --- a/macrobond_data_api/web/web_types/data_package_list_context.py +++ b/macrobond_data_api/web/web_types/data_package_list_context.py @@ -133,13 +133,12 @@ def _parse_body( class DataPackageListContextManager: - _response: Optional["Response"] - def __init__(self, if_modified_since: Optional[datetime], chunk_size: int, webApi: "WebApi") -> None: self._if_modified_since = if_modified_since self.chunk_size = chunk_size self._webApi: Optional["WebApi"] = webApi self._iterator_started = False + self._response: Optional["Response"] = None def __enter__(self) -> DataPackageListContext: params = {} diff --git a/tests/Web/data_package_list_poller/abort.py b/tests/Web/data_package_list_poller/abort.py index 893a6a76..1361b599 100644 --- a/tests/Web/data_package_list_poller/abort.py +++ b/tests/Web/data_package_list_poller/abort.py @@ -6,7 +6,7 @@ from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -80,7 +80,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: raise Exception("should not be called") def on_incremental_begin(self, subscription: "DataPackageBody") -> None: @@ -89,7 +89,10 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: raise Exception("should not be called") @@ -112,10 +115,9 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: hit_test(2) self.abort() - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(3) assert is_aborted is True - assert exception is None api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) @@ -146,10 +148,9 @@ def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataP if hit_test(3, 4) == 4: self.abort() - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(5) assert is_aborted is True - assert exception is None api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) @@ -179,10 +180,9 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(3) - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(4) assert is_aborted is False - assert exception is None self.abort() api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) @@ -218,10 +218,9 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: hit_test(3) self.abort() - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(4) assert is_aborted is True - assert exception is None api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) @@ -260,10 +259,9 @@ def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPa if hit_test(4): self.abort() - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(5) assert is_aborted is True - assert exception is None api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) @@ -302,10 +300,9 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(4) - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(5) assert is_aborted is False - assert exception is None self.abort() api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) @@ -351,10 +348,9 @@ def on_incremental_batch(self, subscription: "DataPackageBody", items: List["Dat if hit_test(4, 6) == 6: self.abort() - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(7) assert is_aborted is True - assert exception is None api = get_api( get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) @@ -397,10 +393,9 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4, 6) - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(7) assert is_aborted is False - assert exception is None self.abort() api = get_api( diff --git a/tests/Web/data_package_list_poller/error.py b/tests/Web/data_package_list_poller/error.py index 1596e501..cf7e682a 100644 --- a/tests/Web/data_package_list_poller/error.py +++ b/tests/Web/data_package_list_poller/error.py @@ -7,7 +7,7 @@ from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -81,7 +81,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: raise Exception("should not be called") def on_incremental_begin(self, subscription: "DataPackageBody") -> None: @@ -90,7 +90,10 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: raise Exception("should not be called") @@ -172,7 +175,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(3) - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(4) raise Exception("Test exception") @@ -285,7 +288,7 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: DataPackageBody, items: List[DataPackageListItem]) -> None: hit_test(4) - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(5) raise Exception("Test exception") @@ -375,7 +378,7 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4, 6) - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(7) raise Exception("Test exception") diff --git a/tests/Web/data_package_list_poller/normal.py b/tests/Web/data_package_list_poller/normal.py index 5b8de3db..24ea70bd 100644 --- a/tests/Web/data_package_list_poller/normal.py +++ b/tests/Web/data_package_list_poller/normal.py @@ -8,7 +8,7 @@ from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -82,7 +82,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: raise Exception("should not be called") def on_incremental_begin(self, subscription: "DataPackageBody") -> None: @@ -91,7 +91,10 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: raise Exception("should not be called") @@ -135,10 +138,9 @@ def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["Da if hit == 5: assert items == [DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6))] - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(6) assert is_aborted is False - assert exception is None api = get_api(get_json_response(DataPackageListState.FULL_LISTING)) @@ -194,10 +196,9 @@ def on_incremental_batch(self, subscription: "DataPackageBody", items: List["Dat else: raise Exception("should not be here") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(7) assert is_aborted is False - assert exception is None api = get_api(get_json_response(DataPackageListState.UP_TO_DATE)) @@ -213,7 +214,7 @@ def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) - # _run_listing and _run_listing_incomplete -def test_listing_and_listing_incomplete() -> None: +def test_listing_and_listing_incomplete_1() -> None: hit = 0 def hit_test(*now: int) -> int: @@ -272,10 +273,9 @@ def on_incremental_batch(self, subscription: "DataPackageBody", items: List["Dat else: raise Exception("should not be here") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(11) assert is_aborted is False - assert exception is None api = get_api( get_json_response(DataPackageListState.INCOMPLETE), get_json_response(DataPackageListState.UP_TO_DATE) @@ -290,3 +290,74 @@ def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) - ).start() assert hit == 12 + + +def test_listing_and_listing_incomplete_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(5, 7, 10) + if hit in (5, 7): + assert secs == self.incomplete_delay + elif hit == 10: + assert secs == self.up_to_date_delay + raise Exception("End of test") + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + if hit_test(3) == 3: + assert subscription.state == DataPackageListState.INCOMPLETE + + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit = hit_test(4, 6, 8) + assert subscription.time_stamp_for_if_modified_since == datetime(2000, 2, 2, 4, 5, 6) + assert subscription.download_full_list_on_or_after == datetime(2000, 2, 1, 4, 5, 6) + + assert items == [ + DataPackageListItem("sek", datetime(2000, 2, 3, 4, 5, 6)), + DataPackageListItem("dkk", datetime(2000, 2, 4, 4, 5, 6)), + DataPackageListItem("usgdp", datetime(2000, 2, 5, 4, 5, 6)), + ] + + if hit in (4, 6): + assert subscription.state == DataPackageListState.INCOMPLETE + elif hit == 8: + assert subscription.state == DataPackageListState.UP_TO_DATE + else: + raise Exception("should not be here") + + def on_incremental_end(self, is_aborted: bool) -> None: + hit_test(9) + assert is_aborted is False + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), + get_json_response(DataPackageListState.INCOMPLETE), + get_json_response(DataPackageListState.UP_TO_DATE), + ) + + with pytest.raises(Exception, match="End of test"): + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 10 diff --git a/tests/Web/data_package_list_poller/on_exception.py b/tests/Web/data_package_list_poller/on_exception.py new file mode 100644 index 00000000..9d81b378 --- /dev/null +++ b/tests/Web/data_package_list_poller/on_exception.py @@ -0,0 +1,378 @@ +from datetime import datetime, timezone +from io import BytesIO +from json import dumps as json_dumps +from typing import Any, List, Optional + +from requests import Response + +from macrobond_data_api.web import WebApi +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource +from macrobond_data_api.web.session import Session +from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState + + +class TestAuth2Session: + __test__ = False + + def __init__(self, *responses: Response): + self.index = 0 + self.responses = responses + + def request(self, *args: Any, **kwargs: Any) -> Response: # pylint: disable=unused-argument + response = self.responses[self.index] + self.index += 1 + return response + + +def get_api(*responses: Response) -> WebApi: + return WebApi(Session("", "", test_auth2_session=TestAuth2Session(*responses))) + + +def get_json_response( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", +) -> Response: + json = json_dumps( + { + "downloadFullListOnOrAfter": downloadFullListOnOrAfter, + "timeStampForIfModifiedSince": timeStampForIfModifiedSince, + "state": state, + "entities": [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ], + } + ) + response = Response() + response.status_code = 200 + response.raw = BytesIO(bytes(json, "utf-8")) + return response + + +def get_broken_json_response( + state: DataPackageListState, + downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", + timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", +) -> Response: + response = get_json_response(state, downloadFullListOnOrAfter, timeStampForIfModifiedSince) + response.raw = BytesIO(bytes(response.raw.getvalue().decode("utf-8")[:-10], "utf-8")) + return response + + +def http_500_response() -> Response: + response = Response() + response.status_code = 500 + return response + + +class TestDataPackageListPoller(DataPackageListPoller): + __test__ = False + + def __init__( + self, + api: WebApi, + download_full_list_on_or_after: Optional[datetime] = None, + time_stamp_for_if_modified_since: Optional[datetime] = None, + chunk_size: int = 200, + ): + super().__init__(api, download_full_list_on_or_after, time_stamp_for_if_modified_since, chunk_size) + self._sleep = self.sleep + self._now = self.now + + def sleep(self, secs: float) -> None: + raise Exception("should not be called") + + def now(self) -> datetime: + raise Exception("should not be called") + + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_full_listing_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: + raise Exception("should not be called") + + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + raise Exception("should not be called") + + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + raise Exception("should not be called") + + +# _run_full_listing + + +def test_full_listing_error_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(2, 3, 4) - 1 + assert secs == self.on_retry_delay * hit + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(5) + assert source is ExceptionSource.FAILED_TO_BEGIN_FULL_LISTING + assert exception is not None + self.abort() + + api = get_api( + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + + _TestDataPackageListPoller(api, chunk_size=200).start() + + assert hit == 5 + + +def test_full_listing_error_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: + hit_test(2) + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(3) + assert source is ExceptionSource.FAILED_TO_GET_BATCH_IN_FULL_LISTING + assert exception is not None + self.abort() + + api = get_api(get_broken_json_response(DataPackageListState.FULL_LISTING)) + + _TestDataPackageListPoller(api, chunk_size=200).start() + + assert hit == 3 + + +# _run_listing + + +def test_listing_error_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def sleep(self, secs: float) -> None: + hit = hit_test(3, 4, 5) - 2 + assert secs == self.on_retry_delay * hit + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(6) + assert source is ExceptionSource.FAILED_TO_BEGIN_LISTING + assert exception is not None + self.abort() + + api = get_api( + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 6 + + +def test_listing_error_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_begin(self, subscription: DataPackageBody) -> None: + hit_test(3) + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(4) + assert source is ExceptionSource.FAILED_TO_GET_BATCH_IN_LISTING + assert exception is not None + self.abort() + + api = get_api( + get_broken_json_response(DataPackageListState.UP_TO_DATE), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 4 + + +# _run_listing and _run_listing_incomplete + + +def test_listing_and_listing_incomplete_error_1() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit = hit_test(5, 6, 7, 8) + if hit == 5: + assert secs == self.incomplete_delay + else: + assert secs == self.on_retry_delay * (hit - 5) + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4) + assert subscription.state == DataPackageListState.INCOMPLETE + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(9) + assert source is ExceptionSource.FAILED_TO_BEGIN_LISTING_INCOMPLETE + assert exception is not None + self.abort() + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), + http_500_response(), + http_500_response(), + http_500_response(), + http_500_response(), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 9 + + +def test_listing_and_listing_incomplete_error_2() -> None: + hit = 0 + + def hit_test(*now: int) -> int: + nonlocal hit + hit += 1 + assert hit in now + return hit + + class _TestDataPackageListPoller(TestDataPackageListPoller): + __test__ = False + + def _test_access(self) -> None: + hit_test(1) + + def sleep(self, secs: float) -> None: + hit_test(5) + assert secs == self.incomplete_delay + + def now(self) -> datetime: + hit_test(2) + return datetime(2000, 1, 1, tzinfo=timezone.utc) + + def on_incremental_begin(self, subscription: "DataPackageBody") -> None: + hit_test(3) + + def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: + hit_test(4) + assert subscription.state == DataPackageListState.INCOMPLETE + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: + hit_test(6) + assert source is ExceptionSource.FAILED_TO_GET_BATCH_IN_LISTING_INCOMPLETE + assert exception is not None + self.abort() + + api = get_api( + get_json_response(DataPackageListState.INCOMPLETE), + get_broken_json_response(DataPackageListState.UP_TO_DATE), + ) + + _TestDataPackageListPoller( + api, + download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), + time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), + ).start() + + assert hit == 6 diff --git a/tests/Web/data_package_list_poller/retry.py b/tests/Web/data_package_list_poller/retry.py index 36daf3ce..0acedc3c 100644 --- a/tests/Web/data_package_list_poller/retry.py +++ b/tests/Web/data_package_list_poller/retry.py @@ -1,13 +1,12 @@ from datetime import datetime, timezone from io import BytesIO from json import dumps as json_dumps -from typing import Any, Dict, List, Optional -import pytest +from typing import Any, List, Optional from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, RetryException +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem, DataPackageListState @@ -33,20 +32,17 @@ def get_json_response( state: DataPackageListState, downloadFullListOnOrAfter: str = "2000-02-01T04:05:06", timeStampForIfModifiedSince: str = "2000-02-02T04:05:06", - entities: Optional[List[Dict[str, str]]] = None, ) -> Response: - if entities is None: - entities = [ - {"name": "sek", "modified": "2000-02-03T04:05:06"}, - {"name": "dkk", "modified": "2000-02-04T04:05:06"}, - {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, - ] json = json_dumps( { "downloadFullListOnOrAfter": downloadFullListOnOrAfter, "timeStampForIfModifiedSince": timeStampForIfModifiedSince, "state": state, - "entities": entities, + "entities": [ + {"name": "sek", "modified": "2000-02-03T04:05:06"}, + {"name": "dkk", "modified": "2000-02-04T04:05:06"}, + {"name": "usgdp", "modified": "2000-02-05T04:05:06"}, + ], } ) response = Response() @@ -87,7 +83,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: raise Exception("should not be called") def on_incremental_begin(self, subscription: "DataPackageBody") -> None: @@ -96,11 +92,13 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: raise Exception("should not be called") -# _run_full_listing def test_full_listing() -> None: hit = 0 @@ -126,7 +124,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(4) - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: hit_test(5) self.abort() @@ -137,38 +135,6 @@ def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) assert hit == 5 -def test_full_listing_error() -> None: - hit = 0 - - def hit_test(*now: int) -> int: - nonlocal hit - hit += 1 - assert hit in now - return hit - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def sleep(self, secs: float) -> None: - hit = hit_test(2, 3, 4) - 1 - assert secs == self.on_retry_delay * hit - - api = get_api( - http_500_response(), - http_500_response(), - http_500_response(), - http_500_response(), - ) - with pytest.raises(RetryException, match="Retry Exception"): - _TestDataPackageListPoller(api, chunk_size=200).start() - - assert hit == 4 - - -# _run_listing def test_listing() -> None: hit = 0 @@ -198,7 +164,7 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: hit_test(5) - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(6) self.abort() @@ -213,46 +179,6 @@ def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) - assert hit == 6 -def test_listing_error() -> None: - hit = 0 - - def hit_test(*now: int) -> int: - nonlocal hit - hit += 1 - assert hit in now - return hit - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def now(self) -> datetime: - hit_test(2) - return datetime(2000, 1, 1, tzinfo=timezone.utc) - - def sleep(self, secs: float) -> None: - hit = hit_test(3, 4, 5) - 2 - assert secs == self.on_retry_delay * hit - - api = get_api( - http_500_response(), - http_500_response(), - http_500_response(), - http_500_response(), - ) - - with pytest.raises(RetryException, match="Retry Exception"): - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - ).start() - - assert hit == 5 - - # _run_listing and _run_listing_incomplete def test_listing_and_listing_incomplete() -> None: hit = 0 @@ -292,7 +218,7 @@ def on_incremental_batch(self, subscription: "DataPackageBody", items: List["Dat elif hit == 7: assert subscription.state == DataPackageListState.UP_TO_DATE - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: hit_test(8) self.abort() @@ -309,59 +235,3 @@ def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) - ).start() assert hit == 8 - - -def test_listing_and_listing_incomplete_error() -> None: - hit = 0 - - def hit_test(*now: int) -> int: - nonlocal hit - hit += 1 - assert hit in now - return hit - - class _TestDataPackageListPoller(TestDataPackageListPoller): - __test__ = False - - def _test_access(self) -> None: - hit_test(1) - - def sleep(self, secs: float) -> None: - hit = hit_test(5, 6, 7, 8) - if hit == 5: - assert secs == self.incomplete_delay - else: - assert secs == self.on_retry_delay * (hit - 5) - - def now(self) -> datetime: - hit_test(2) - return datetime(2000, 1, 1, tzinfo=timezone.utc) - - def on_incremental_begin(self, subscription: "DataPackageBody") -> None: - hit_test(3) - - def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: - hit_test(4) - assert subscription.state == DataPackageListState.INCOMPLETE - - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: - hit_test(9) - self.abort() - assert is_aborted is False - assert exception is not None - - api = get_api( - get_json_response(DataPackageListState.INCOMPLETE), - http_500_response(), - http_500_response(), - http_500_response(), - http_500_response(), - ) - - _TestDataPackageListPoller( - api, - download_full_list_on_or_after=datetime(3000, 1, 1, tzinfo=timezone.utc), - time_stamp_for_if_modified_since=datetime(1000, 1, 1, tzinfo=timezone.utc), - ).start() - - assert hit == 9 diff --git a/tests/Web/data_package_list_poller/test_access.py b/tests/Web/data_package_list_poller/test_access.py index e8bf5b21..36cc061a 100644 --- a/tests/Web/data_package_list_poller/test_access.py +++ b/tests/Web/data_package_list_poller/test_access.py @@ -6,7 +6,7 @@ from requests import Response from macrobond_data_api.web import WebApi -from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller +from macrobond_data_api.web.data_package_list_poller import DataPackageListPoller, ExceptionSource from macrobond_data_api.web.session import Session from macrobond_data_api.web.web_types import DataPackageBody, DataPackageListItem @@ -50,7 +50,7 @@ def on_full_listing_begin(self, subscription: "DataPackageBody") -> None: def on_full_listing_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_full_listing_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_full_listing_end(self, is_aborted: bool) -> None: raise Exception("should not be called") def on_incremental_begin(self, subscription: "DataPackageBody") -> None: @@ -59,7 +59,10 @@ def on_incremental_begin(self, subscription: "DataPackageBody") -> None: def on_incremental_batch(self, subscription: "DataPackageBody", items: List["DataPackageListItem"]) -> None: raise Exception("should not be called") - def on_incremental_end(self, is_aborted: bool, exception: Optional[Exception]) -> None: + def on_incremental_end(self, is_aborted: bool) -> None: + raise Exception("should not be called") + + def on_exception(self, source: ExceptionSource, exception: Exception) -> None: raise Exception("should not be called")