✨ Resume build log stream if interrupted

Author: buurro

requirements-tests.txt

@@ -5,3 +5,4 @@ coverage[toml] >=6.2,<8.0
 mypy ==1.14.1
 ruff ==0.13.0
 respx ==0.22.0
+time-machine ==2.19.0

src/fastapi_cloud_cli/commands/deploy.py

@@ -1,5 +1,4 @@
 import contextlib
-import json
 import logging
 import subprocess
 import tarfile
@@ -8,7 +7,7 @@
 from enum import Enum
 from itertools import cycle
 from pathlib import Path
-from typing import Any, Dict, Generator, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 import rignore
 import typer
@@ -20,7 +19,7 @@
 from typing_extensions import Annotated
 
 from fastapi_cloud_cli.commands.login import login
-from fastapi_cloud_cli.utils.api import APIClient
+from fastapi_cloud_cli.utils.api import APIClient, BuildLogError, BuildLogType
 from fastapi_cloud_cli.utils.apps import AppConfig, get_app_config, write_app_config
 from fastapi_cloud_cli.utils.auth import is_logged_in
 from fastapi_cloud_cli.utils.cli import get_rich_toolkit, handle_http_errors
@@ -225,21 +224,11 @@ def _get_apps(team_id: str) -> List[AppResponse]:
     return [AppResponse.model_validate(app) for app in data]
 
 
-def _stream_build_logs(deployment_id: str) -> Generator[str, None, None]:
-    with APIClient() as client:
-        with client.stream(
-            "GET", f"/deployments/{deployment_id}/build-logs", timeout=60
-        ) as response:
-            response.raise_for_status()
-
-            yield from response.iter_lines()
-
-
 WAITING_MESSAGES = [
     "🚀 Preparing for liftoff! Almost there...",
     "👹 Sneaking past the dependency gremlins... Don't wake them up!",
     "🤏 Squishing code into a tiny digital sandwich. Nom nom nom.",
-    "📉 Server space running low. Time to delete those cat videos?",
+    "🐱 Removing cat videos from our servers to free up space.",
     "🐢 Uploading at blazing speeds of 1 byte per hour. Patience, young padawan.",
     "🔌 Connecting to server... Please stand by while we argue with the firewall.",
     "💥 Oops! We've angered the Python God. Sacrificing a rubber duck to appease it.",
@@ -350,43 +339,50 @@ def _wait_for_deployment(
     with toolkit.progress(
         next(messages), inline_logs=True, lines_to_show=20
     ) as progress:
-        with handle_http_errors(progress=progress):
-            for line in _stream_build_logs(deployment.id):
-                time_elapsed = time.monotonic() - started_at
+        with APIClient() as client:
+            try:
+                for log in client.stream_build_logs(deployment.id):
+                    time_elapsed = time.monotonic() - started_at
 
-                data = json.loads(line)
+                    if log.type == BuildLogType.message and log.message:
+                        progress.log(Text.from_ansi(log.message.rstrip()))
 
-                if "message" in data:
-                    progress.log(Text.from_ansi(data["message"].rstrip()))
+                    if log.type == BuildLogType.complete:
+                        progress.log("")
+                        progress.log(
+                            f"🐔 Ready the chicken! Your app is ready at [link={deployment.url}]{deployment.url}[/link]"
+                        )
 
-                if data.get("type") == "complete":
-                    progress.log("")
-                    progress.log(
-                        f"🐔 Ready the chicken! Your app is ready at [link={deployment.url}]{deployment.url}[/link]"
-                    )
+                        progress.log("")
 
-                    progress.log("")
+                        progress.log(
+                            f"You can also check the app logs at [link={deployment.dashboard_url}]{deployment.dashboard_url}[/link]"
+                        )
 
-                    progress.log(
-                        f"You can also check the app logs at [link={deployment.dashboard_url}]{deployment.dashboard_url}[/link]"
-                    )
+                        break
 
-                    break
+                    if log.type == BuildLogType.failed:
+                        progress.log("")
+                        progress.log(
+                            f"😔 Oh no! Something went wrong. Check out the logs at [link={deployment.dashboard_url}]{deployment.dashboard_url}[/link]"
+                        )
+                        raise typer.Exit(1)
 
-                if data.get("type") == "failed":
-                    progress.log("")
-                    progress.log(
-                        f"😔 Oh no! Something went wrong. Check out the logs at [link={deployment.dashboard_url}]{deployment.dashboard_url}[/link]"
-                    )
-                    raise typer.Exit(1)
+                    if time_elapsed > 30:
+                        messages = cycle(LONG_WAIT_MESSAGES)
 
-                if time_elapsed > 30:
-                    messages = cycle(LONG_WAIT_MESSAGES)  # pragma: no cover
+                    if (time.monotonic() - last_message_changed_at) > 2:
+                        progress.title = next(messages)
 
-                if (time.monotonic() - last_message_changed_at) > 2:
-                    progress.title = next(messages)  # pragma: no cover
+                        last_message_changed_at = time.monotonic()
 
-                    last_message_changed_at = time.monotonic()  # pragma: no cover
+            except BuildLogError as e:
+                logger.error("Build log streaming failed: %s", e)
+                toolkit.print_line()
+                toolkit.print(
+                    f"⚠️  Unable to stream build logs. Check the dashboard for status: [link={deployment.dashboard_url}]{deployment.dashboard_url}[/link]"
+                )
+                raise typer.Exit(1) from e
 
 
 class SignupToWaitingList(BaseModel):

src/fastapi_cloud_cli/utils/api.py

@@ -1,9 +1,95 @@
+import logging
+import time
+from contextlib import AbstractContextManager, contextmanager
+from datetime import timedelta
+from enum import Enum
+from typing import Generator, Optional
+
 import httpx
+from pydantic import BaseModel, ValidationError
 
 from fastapi_cloud_cli import __version__
 from fastapi_cloud_cli.config import Settings
 from fastapi_cloud_cli.utils.auth import get_auth_token
 
+logger = logging.getLogger(__name__)
+
+BUILD_LOG_MAX_RETRIES = 3
+BUILD_LOG_TIMEOUT = timedelta(minutes=5)
+
+
+class BuildLogError(Exception): ...
+
+
+class BuildLogType(str, Enum):
+    message = "message"
+    complete = "complete"
+    failed = "failed"
+    timeout = "timeout"  # Request closed, reconnect to continue
+    heartbeat = "heartbeat"  # Keepalive signal when no new logs
+
+
+class BuildLogLine(BaseModel):
+    type: BuildLogType
+    message: str | None = None
+    id: str | None = None
+
+
+@contextmanager
+def attempt(attempt_number: int) -> Generator[None, None, None]:
+    def _backoff() -> None:
+        backoff_seconds = min(2**attempt_number, 30)
+        logger.debug(
+            "Retrying in %ds (attempt %d)",
+            backoff_seconds,
+            attempt_number,
+        )
+        time.sleep(backoff_seconds)
+
+    try:
+        yield
+
+    except (
+        httpx.TimeoutException,
+        httpx.NetworkError,
+        httpx.RemoteProtocolError,
+    ) as error:
+        logger.debug("Network error (will retry): %s", error)
+
+        _backoff()
+
+    except httpx.HTTPStatusError as error:
+        if error.response.status_code >= 500:
+            logger.debug(
+                "Server error %d (will retry): %s",
+                error.response.status_code,
+                error,
+            )
+            _backoff()
+        else:
+            # Try to get response text, but handle streaming responses gracefully
+            try:
+                error_detail = error.response.text
+            except Exception:
+                error_detail = "(response body unavailable)"
+            raise BuildLogError(
+                f"HTTP {error.response.status_code}: {error_detail}"
+            ) from error
+
+
+def attempts(
+    total_attempts: int = 3, timeout: timedelta = timedelta(minutes=5)
+) -> Generator[AbstractContextManager[None], None, None]:
+    start = time.monotonic()
+
+    for attempt_number in range(total_attempts):
+        if time.monotonic() - start > timeout.total_seconds():
+            raise TimeoutError(
+                "Build log streaming timed out after %ds", timeout.total_seconds()
+            )
+
+        yield attempt(attempt_number)
+
 
 class APIClient(httpx.Client):
     def __init__(self) -> None:
@@ -19,3 +105,62 @@ def __init__(self) -> None:
                 "User-Agent": f"fastapi-cloud-cli/{__version__}",
             },
         )
+
+    def stream_build_logs(
+        self, deployment_id: str
+    ) -> Generator[BuildLogLine, None, None]:
+        last_id = None
+
+        for attempt in attempts(BUILD_LOG_MAX_RETRIES, BUILD_LOG_TIMEOUT):
+            with attempt:
+                while True:
+                    params = {"last_id": last_id} if last_id else None
+
+                    with self.stream(
+                        "GET",
+                        f"/deployments/{deployment_id}/build-logs",
+                        timeout=60,
+                        params=params,
+                    ) as response:
+                        response.raise_for_status()
+
+                        for line in response.iter_lines():
+                            if not line or not line.strip():
+                                continue
+
+                            if log_line := self._parse_log_line(line):
+                                if log_line.id:
+                                    last_id = log_line.id
+
+                                if log_line.type == BuildLogType.message:
+                                    yield log_line
+
+                                if log_line.type in (
+                                    BuildLogType.complete,
+                                    BuildLogType.failed,
+                                ):
+                                    yield log_line
+
+                                    return
+
+                                if log_line.type == BuildLogType.timeout:
+                                    logger.debug("Received timeout; reconnecting")
+                                    break  # Breaks for loop to reconnect
+
+                        else:  # Only triggered if the for loop is not broken
+                            logger.debug(
+                                "Connection closed by server unexpectedly; attempting to reconnect"
+                            )
+                            break
+
+                    time.sleep(0.5)
+
+        # Exhausted retries without getting any response
+        raise BuildLogError(f"Failed after {BUILD_LOG_MAX_RETRIES} attempts")
+
+    def _parse_log_line(self, line: str) -> Optional[BuildLogLine]:
+        try:
+            return BuildLogLine.model_validate_json(line)
+        except ValidationError as e:
+            logger.debug("Skipping malformed log: %s (error: %s)", line[:100], e)
+            return None