From 7b59e5b2df0356e48a16dd3b4af0cc3c1544dd91 Mon Sep 17 00:00:00 2001 From: Gaurav Date: Wed, 21 Oct 2026 14:11:04 -0400 Subject: [PATCH 1/6] fix(sessions)!: make session retirement permanent Persist retirement independently of error scores and cover session and pool restoration. BREAKING CHANGE: Retired sessions cannot become usable again after mark_good(). --- docs/upgrading/upgrading_to_v2.md | 6 ++++ src/crawlee/sessions/_models.py | 1 + src/crawlee/sessions/_session.py | 14 +++++--- tests/unit/sessions/test_session.py | 44 +++++++++++++++++++++++ tests/unit/sessions/test_session_pool.py | 46 ++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 4 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 894d2a8567..f221cda01e 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -5,6 +5,12 @@ title: Upgrading to v2 This page summarizes the breaking changes between Crawlee for Python v1 and v2. +## Session retirement is permanent + +`Session.retire()` permanently blocks the session. Calling `mark_good()` can still lower its error score, but cannot make it usable again. Create a new session to replace a retired one. Repeated calls to `retire()` leave the error score unchanged. + +Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits. + ## Python version support We drop support for Python 3.10, which reached its end of life in October 2026. The minimum supported version is now Python 3.11. diff --git a/src/crawlee/sessions/_models.py b/src/crawlee/sessions/_models.py index 1cb3be1a1b..93b36fc452 100644 --- a/src/crawlee/sessions/_models.py +++ b/src/crawlee/sessions/_models.py @@ -27,6 +27,7 @@ class SessionModel(BaseModel): usage_count: int max_usage_count: int error_score: float + retired: bool = False cookies: list[CookieParam] blocked_status_codes: list[int] diff --git a/src/crawlee/sessions/_session.py b/src/crawlee/sessions/_session.py index 733ca46d25..4502a00507 100644 --- a/src/crawlee/sessions/_session.py +++ b/src/crawlee/sessions/_session.py @@ -45,6 +45,7 @@ def __init__( usage_count: int = 0, max_usage_count: int = 50, error_score: float = 0.0, + retired: bool = False, cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None, blocked_status_codes: list | None = None, ) -> None: @@ -60,6 +61,7 @@ def __init__( usage_count: Number of times the session has been used. max_usage_count: Maximum allowable uses of the session before it is considered expired. error_score: Current error score of the session. + retired: Whether the session has been permanently retired. cookies: Cookies associated with the session. blocked_status_codes: HTTP status codes that indicate a session should be blocked. """ @@ -72,6 +74,7 @@ def __init__( self._usage_count = usage_count self._max_usage_count = max_usage_count self._error_score = error_score + self._retired = retired self._cookies = SessionCookies(cookies) or SessionCookies() self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES) @@ -145,8 +148,8 @@ def expires_at(self) -> datetime: @property def is_blocked(self) -> bool: - """Indicate whether the session is blocked based on the error score..""" - return self._error_score >= self._max_error_score + """Indicate whether the session is retired or blocked by its error score.""" + return self._retired or self._error_score >= self._max_error_score @property def is_expired(self) -> bool: @@ -183,6 +186,7 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: usage_count=self._usage_count, max_usage_count=self._max_usage_count, error_score=self._error_score, + retired=self._retired, cookies=self._cookies.get_cookies_as_dicts(), blocked_status_codes=list(self._blocked_status_codes), ) @@ -211,14 +215,16 @@ def mark_bad(self) -> None: self.retire() def retire(self) -> None: - """Retire the session by setting the error score to the maximum value. + """Permanently retire the session and increase its error score once. This method should be used if the session usage was unsuccessful and you are sure that it is because of the session configuration and not any external matters. For example when server returns 403 status code. If the session does not work due to some external factors as server error such as 5XX you probably want to use `mark_bad` method. """ - self._error_score += self._max_error_score + if not self._retired: + self._retired = True + self._error_score += self._max_error_score def is_blocked_status_code( self, diff --git a/tests/unit/sessions/test_session.py b/tests/unit/sessions/test_session.py index e8a8559070..6edb801b47 100644 --- a/tests/unit/sessions/test_session.py +++ b/tests/unit/sessions/test_session.py @@ -5,6 +5,7 @@ import pytest from crawlee.sessions._cookies import SessionCookies +from crawlee.sessions._models import SessionModel from crawlee.sessions._session import Session @@ -89,6 +90,49 @@ def test_retire_method(session: Session) -> None: assert session.usage_count == initial_usage_count +def test_retired_session_stays_blocked_after_success() -> None: + """A successful request cannot make a retired session usable again.""" + session = Session() + session.retire() + session.mark_good() + + assert session.error_score < 3.0 + assert session.is_blocked + assert not session.is_usable + + +def test_retirement_survives_serialization() -> None: + """Retirement survives a JSON round trip even with a reduced error score.""" + session = Session() + session.retire() + session.mark_good() + state = session.get_state(as_dict=False).model_dump_json(by_alias=True) + restored = Session.from_model(SessionModel.model_validate_json(state)) + restored.mark_good() + + assert restored.is_blocked + assert not restored.is_usable + + +def test_session_without_retirement_field_can_be_restored() -> None: + """Saved sessions without a retirement field remain loadable.""" + state = Session().get_state(as_dict=True) + state.pop('retired', None) + restored = Session.from_model(SessionModel.model_validate(state)) + + assert restored.is_usable + + +def test_retire_is_idempotent() -> None: + """Repeated retirement leaves the error score and usage count unchanged.""" + session = Session() + session.retire() + state = session.get_state(as_dict=True) + session.retire() + + assert session.get_state(as_dict=True) == state + + def test_mark_good_at_usage_limit_no_double_increment() -> None: """Test that mark_good at max usage count does not double-increment usage_count via retire.""" session = Session(max_usage_count=5, usage_count=4) diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py index 166f517a2f..44ba38853f 100644 --- a/tests/unit/sessions/test_session_pool.py +++ b/tests/unit/sessions/test_session_pool.py @@ -83,6 +83,19 @@ async def test_get_session(session_pool: SessionPool) -> None: assert session.is_usable +async def test_pool_does_not_reuse_retired_session_after_success() -> None: + """Neither pool lookup returns a retired session after a successful request.""" + async with SessionPool(max_pool_size=1, persistence_enabled=False) as pool: + session = await pool.get_session() + session.retire() + session.mark_good() + + assert await pool.get_session_by_id(session.id) is None + replacement = await pool.get_session() + assert replacement.id != session.id + assert replacement.is_usable + + async def test_get_session_no_usable(caplog: pytest.LogCaptureFixture, session_pool: SessionPool) -> None: """Ensure that retrieval of a non-existent or retired session returns None and logs warning.""" session = await session_pool.get_session_by_id('non_existent') @@ -170,6 +183,39 @@ async def test_session_pool_persist_and_restore(event_manager: EventManager, kvs assert previous_state is None +async def test_retired_session_is_not_restored(event_manager: EventManager, kvs: KeyValueStore) -> None: + """A saved retired session is excluded when the pool is reopened.""" + service_locator.set_event_manager(event_manager) + + async with SessionPool( + max_pool_size=1, + persistence_enabled=True, + persist_state_kvs_name=KVS_NAME, + persist_state_key=PERSIST_STATE_KEY, + ) as pool: + session = await pool.get_session() + session.retire() + session.mark_good() + event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False)) + await event_manager.wait_for_all_listeners_to_complete() + + saved_state = await kvs.get_value(PERSIST_STATE_KEY) + assert isinstance(saved_state, dict) + saved_pool = SessionPoolModel.model_validate(saved_state) + assert not saved_pool.sessions[session.id].is_usable + + async with SessionPool( + max_pool_size=1, + persistence_enabled=True, + persist_state_kvs_name=KVS_NAME, + persist_state_key=PERSIST_STATE_KEY, + ) as restored_pool: + assert await restored_pool.get_session_by_id(session.id) is None + replacement = await restored_pool.get_session() + assert replacement.id != session.id + assert replacement.is_usable + + async def test_methods_raise_error_when_not_active() -> None: session = Session() session_pool = SessionPool() From 261dab6f8f54f21c27092703703fdd412880370a Mon Sep 17 00:00:00 2001 From: Gaurav Date: Mon, 26 Oct 2026 14:15:46 -0400 Subject: [PATCH 2/6] fix(sessions): separate retirement from blocking scores --- docs/upgrading/upgrading_to_v2.md | 14 ++++--- src/crawlee/sessions/_session.py | 18 ++++++--- tests/unit/sessions/test_session.py | 62 +++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 19 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index f221cda01e..ec98e36859 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -5,12 +5,6 @@ title: Upgrading to v2 This page summarizes the breaking changes between Crawlee for Python v1 and v2. -## Session retirement is permanent - -`Session.retire()` permanently blocks the session. Calling `mark_good()` can still lower its error score, but cannot make it usable again. Create a new session to replace a retired one. Repeated calls to `retire()` leave the error score unchanged. - -Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits. - ## Python version support We drop support for Python 3.10, which reached its end of life in October 2026. The minimum supported version is now Python 3.11. @@ -173,3 +167,11 @@ current concurrency 3, desired 5, overloaded: cpu 58% of the time (limit 40%) To read the concurrency values in code, use `crawler.concurrency_system`. The autoscaling loggers moved together with the package. `crawlee._autoscaling.autoscaled_pool`, `crawlee._autoscaling.snapshotter` and `crawlee._autoscaling.system_status` are now `crawlee.autoscaling._autoscaled_pool`, `crawlee.autoscaling._snapshotter` and `crawlee.autoscaling._system_status`. The state log, along with the messages about scaling and free capacity, now comes from the new `crawlee.autoscaling._concurrency_system` logger. To configure all of them at once, use the `crawlee.autoscaling` logger. + +## Session retirement is permanent + +`Session.retire()` permanently retires the session. Calling `mark_good()` on a retired session leaves its error score and usage count unchanged. Create a new session to replace a retired one. Repeated calls to `retire()` leave the error score unchanged. + +The `Session.retired` property indicates whether a session has been retired. Sessions that expire or reach their usage limit retire without an additional error score penalty, so `is_blocked` continues to indicate whether the error score reached its limit. + +Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits. diff --git a/src/crawlee/sessions/_session.py b/src/crawlee/sessions/_session.py index 4502a00507..4389060cef 100644 --- a/src/crawlee/sessions/_session.py +++ b/src/crawlee/sessions/_session.py @@ -141,6 +141,11 @@ def usage_count(self) -> float: """Get the current usage count.""" return self._usage_count + @property + def retired(self) -> bool: + """Indicate whether the session has been permanently retired.""" + return self._retired + @property def expires_at(self) -> datetime: """Get the expiration datetime of the session.""" @@ -148,8 +153,8 @@ def expires_at(self) -> datetime: @property def is_blocked(self) -> bool: - """Indicate whether the session is retired or blocked by its error score.""" - return self._retired or self._error_score >= self._max_error_score + """Indicate whether the session is blocked by its error score.""" + return self._error_score >= self._max_error_score @property def is_expired(self) -> bool: @@ -164,7 +169,7 @@ def is_max_usage_count_reached(self) -> bool: @property def is_usable(self) -> bool: """Determine if the session is usable for next requests.""" - return not (self.is_blocked or self.is_expired or self.is_max_usage_count_reached) + return not (self._retired or self.is_blocked or self.is_expired or self.is_max_usage_count_reached) @overload def get_state(self, *, as_dict: Literal[True]) -> dict: ... @@ -196,6 +201,9 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: def mark_good(self) -> None: """Mark the session as good. Should be called after a successful session usage.""" + if self._retired: + return + self._usage_count += 1 if self._error_score > 0: @@ -203,7 +211,7 @@ def mark_good(self) -> None: # Retire the session if it is not usable anymore if not self.is_usable: - self.retire() + self._retired = True def mark_bad(self) -> None: """Mark the session as bad after an unsuccessful session usage.""" @@ -212,7 +220,7 @@ def mark_bad(self) -> None: # Retire the session if it is not usable anymore if not self.is_usable: - self.retire() + self._retired = True def retire(self) -> None: """Permanently retire the session and increase its error score once. diff --git a/tests/unit/sessions/test_session.py b/tests/unit/sessions/test_session.py index 6edb801b47..7e434e6c04 100644 --- a/tests/unit/sessions/test_session.py +++ b/tests/unit/sessions/test_session.py @@ -36,6 +36,7 @@ def test_session_init(session: Session) -> None: assert not session.is_expired assert not session.is_max_usage_count_reached assert session.is_usable + assert not session.retired def test_session_get_state(session: Session) -> None: @@ -90,37 +91,82 @@ def test_retire_method(session: Session) -> None: assert session.usage_count == initial_usage_count -def test_retired_session_stays_blocked_after_success() -> None: - """A successful request cannot make a retired session usable again.""" +def test_mark_good_leaves_retired_session_unchanged() -> None: + """A successful request leaves a retired session's state unchanged.""" session = Session() session.retire() + state = session.get_state(as_dict=True) session.mark_good() - assert session.error_score < 3.0 + assert session.get_state(as_dict=True) == state + assert session.retired assert session.is_blocked assert not session.is_usable def test_retirement_survives_serialization() -> None: """Retirement survives a JSON round trip even with a reduced error score.""" - session = Session() - session.retire() - session.mark_good() + session = Session(retired=True, error_score=1.0) state = session.get_state(as_dict=False).model_dump_json(by_alias=True) restored = Session.from_model(SessionModel.model_validate_json(state)) restored.mark_good() - assert restored.is_blocked + assert restored.retired + assert restored.error_score == 1.0 + assert restored.usage_count == 0 + assert not restored.is_blocked assert not restored.is_usable def test_session_without_retirement_field_can_be_restored() -> None: """Saved sessions without a retirement field remain loadable.""" state = Session().get_state(as_dict=True) - state.pop('retired', None) + del state['retired'] restored = Session.from_model(SessionModel.model_validate(state)) assert restored.is_usable + assert not restored.retired + + +@pytest.mark.parametrize( + 'mark_method', + [pytest.param('mark_good', id='successful-request'), pytest.param('mark_bad', id='failed-request')], +) +@pytest.mark.parametrize( + 'expired', + [pytest.param(False, id='usage-limit'), pytest.param(True, id='expired')], +) +def test_automatic_retirement_preserves_error_score(mark_method: str, *, expired: bool) -> None: + """Expiration and usage limits retire sessions without adding a blocking penalty.""" + session = Session( + error_score=0.5, + max_usage_count=10 if expired else 1, + created_at=datetime.now(UTC) - timedelta(hours=1) if expired else None, + ) + getattr(session, mark_method)() + + assert session.retired + assert not session.is_usable + assert not session.is_blocked + assert session.error_score == (0.0 if mark_method == 'mark_good' else 1.5) + assert session.usage_count == 1 + + state = session.get_state(as_dict=True) + session.mark_good() + session.retire() + assert session.get_state(as_dict=True) == state + + +def test_automatic_retirement_at_error_limit_preserves_score() -> None: + """Reaching the error limit retires a session without a second error penalty.""" + session = Session(max_error_score=1.0) + session.mark_bad() + + assert session.retired + assert session.is_blocked + assert not session.is_usable + assert session.error_score == 1.0 + assert session.usage_count == 1 def test_retire_is_idempotent() -> None: From 5af765778a15550a98dc8466a57cfa02e71219f8 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 29 Oct 2026 16:45:17 +0100 Subject: [PATCH 3/6] docs(sessions): Clarify how retirement affects the error score --- docs/upgrading/upgrading_to_v2.md | 4 ++-- src/crawlee/sessions/_session.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 2473fc4efa..68301046e2 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -275,8 +275,8 @@ The two DEBUG messages that `RequestQueue.is_finished` logged, "Background add r ## Session retirement is permanent -`Session.retire()` permanently retires the session. Calling `mark_good()` on a retired session leaves its error score and usage count unchanged. Create a new session to replace a retired one. Repeated calls to `retire()` leave the error score unchanged. +`Session.retire()` permanently retires the session. Calling `mark_good()` on a retired session leaves its error score and usage count unchanged. Create a new session to replace a retired one. Calling `retire()` on a session that is already retired leaves its error score unchanged. -The `Session.retired` property indicates whether a session has been retired. Sessions that expire or reach their usage limit retire without an additional error score penalty, so `is_blocked` continues to indicate whether the error score reached its limit. +The `Session.retired` property indicates whether a session has been retired. Sessions that expire, reach their usage limit, or reach their error score limit through `mark_bad()` retire without an additional `max_error_score` penalty, so `is_blocked` continues to indicate whether the error score reached its limit. Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits. diff --git a/src/crawlee/sessions/_session.py b/src/crawlee/sessions/_session.py index 4389060cef..2f4498536c 100644 --- a/src/crawlee/sessions/_session.py +++ b/src/crawlee/sessions/_session.py @@ -200,7 +200,10 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: return model def mark_good(self) -> None: - """Mark the session as good. Should be called after a successful session usage.""" + """Mark the session as good. Should be called after a successful session usage. + + A retired session is left unchanged. + """ if self._retired: return @@ -223,7 +226,7 @@ def mark_bad(self) -> None: self._retired = True def retire(self) -> None: - """Permanently retire the session and increase its error score once. + """Permanently retire the session and add `max_error_score` to its error score, unless it is already retired. This method should be used if the session usage was unsuccessful and you are sure that it is because of the session configuration and not any external matters. For example when server returns 403 status code. From cdb5a23099308c13d6a38b700040c6cd120529ce Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 29 Oct 2026 16:45:18 +0100 Subject: [PATCH 4/6] test(sessions): Assert the retired flag is persisted --- tests/unit/sessions/test_session_pool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py index 44ba38853f..12561543d8 100644 --- a/tests/unit/sessions/test_session_pool.py +++ b/tests/unit/sessions/test_session_pool.py @@ -202,6 +202,7 @@ async def test_retired_session_is_not_restored(event_manager: EventManager, kvs: saved_state = await kvs.get_value(PERSIST_STATE_KEY) assert isinstance(saved_state, dict) saved_pool = SessionPoolModel.model_validate(saved_state) + assert saved_pool.sessions[session.id].retired assert not saved_pool.sessions[session.id].is_usable async with SessionPool( From ce27d9b9cd3f3216b24e56789c4e17b8e628bc94 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 29 Oct 2026 16:45:19 +0100 Subject: [PATCH 5/6] docs: Retire the session last in the standalone session example --- .../code_examples/session_management/sm_standalone.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guides/code_examples/session_management/sm_standalone.py b/docs/guides/code_examples/session_management/sm_standalone.py index 32989dc7e0..72bd942fb6 100644 --- a/docs/guides/code_examples/session_management/sm_standalone.py +++ b/docs/guides/code_examples/session_management/sm_standalone.py @@ -14,12 +14,12 @@ async def main() -> None: # Increase the error_score. session.mark_bad() - # Throw away the session. - session.retire() - # Lower the error_score and mark the session good. session.mark_good() + # Throw away the session. + session.retire() + if __name__ == '__main__': asyncio.run(main()) From 4b6925ea0c7a68412943ffdb59e67bc5a58af6c5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 29 Oct 2026 16:49:06 +0100 Subject: [PATCH 6/6] docs(sessions): Describe the v1 retirement behavior in the upgrade guide --- docs/upgrading/upgrading_to_v2.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 68301046e2..15c806ac27 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -275,8 +275,10 @@ The two DEBUG messages that `RequestQueue.is_finished` logged, "Background add r ## Session retirement is permanent -`Session.retire()` permanently retires the session. Calling `mark_good()` on a retired session leaves its error score and usage count unchanged. Create a new session to replace a retired one. Calling `retire()` on a session that is already retired leaves its error score unchanged. +In v1, `Session.retire()` only added `max_error_score` to the error score, so a later `mark_good()` could lower it again and make the retired session usable. In v2, a separate `retired` flag tracks retirement, exposed as the `Session.retired` property, and a retired session is never usable again. Create a new session to replace it. -The `Session.retired` property indicates whether a session has been retired. Sessions that expire, reach their usage limit, or reach their error score limit through `mark_bad()` retire without an additional `max_error_score` penalty, so `is_blocked` continues to indicate whether the error score reached its limit. +On a retired session, `mark_good()` does nothing, so its error score and usage count stay as they are. Calling `retire()` on a session that is already retired also leaves the error score unchanged. -Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits. +When `mark_good()` or `mark_bad()` retires a session because it expired, reached its usage limit, or reached its error score limit, it only sets the flag. In v1, it also added `max_error_score`. As a result, such sessions have a lower error score in v2, and `is_blocked` is `True` only when the error score itself reached `max_error_score`. + +Session state, including the persisted session pool state, has a new `retired` field. Sessions saved by v1 load with `retired=False` and keep their error score.