diff --git a/.github/workflows/fireworks-tracing-tests.yml b/.github/workflows/fireworks-tracing-tests.yml index 867fef1d..b66acff7 100644 --- a/.github/workflows/fireworks-tracing-tests.yml +++ b/.github/workflows/fireworks-tracing-tests.yml @@ -1,15 +1,8 @@ name: Fireworks Tracing Tests +# These end-to-end tests run against the hosted Fireworks tracing gateway (https://tracing.fireworks.ai), +# which was shut down on October 28, 2026, so they no longer run on push or pull_request. on: - push: - branches: [main] - paths-ignore: - - "docs/**" - - "*.md" - pull_request: # Run on all pull requests - paths-ignore: - - "docs/**" - - "*.md" workflow_dispatch: # Allow manual triggering jobs: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f65707c..8609440f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - (Your new features here) ### Changed -- (Your changes in existing functionality here) +- `FireworksTracingHttpHandler` is no longer attached automatically by `init_external_logging_from_env()` (used by `@evaluation_test`) when `FIREWORKS_API_KEY` is set; set `FW_TRACING_GATEWAY_BASE_URL` (or `GATEWAY_URL`) to opt in. ### Deprecated -- (Your soon-to-be removed features here) +- The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. These features depend on it, now emit a `DeprecationWarning`, and will be removed in a future release: `RemoteRolloutProcessor`, `GithubActionRolloutProcessor`, `FireworksTracingAdapter`, `FireworksTracingHttpHandler`, and the Fireworks backend of `ep logs`. In the TypeScript package, `FireworksTransport`, `logger`, and `createRolloutLogger` are marked `@deprecated`, and `FireworksTransport` warns once when it sends to the hosted URL. Until removal they only work with a self-hosted gateway that implements the same API, configured explicitly. ### Removed - (Your now removed features here) diff --git a/README.md b/README.md index 87d7d280..d25622c2 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Eval Protocol makes this possible in two ways: The result: RL that works out-of-the-box for existing production agents. +> **Deprecation notice:** The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. SDK features that depend on it (`RemoteRolloutProcessor`, `GithubActionRolloutProcessor`, `FireworksTracingAdapter`, `FireworksTracingHttpHandler`, and the Fireworks backend of `ep logs`) are deprecated and will be removed in a future release. See the [changelog](CHANGELOG.md). + ## Who This Is For - **Applied AI teams** adding RL to existing production agents. diff --git a/eval_protocol/adapters/fireworks_tracing.py b/eval_protocol/adapters/fireworks_tracing.py index 0ff86f71..8bc89769 100644 --- a/eval_protocol/adapters/fireworks_tracing.py +++ b/eval_protocol/adapters/fireworks_tracing.py @@ -8,12 +8,14 @@ import asyncio import json import logging +import warnings import aiohttp import requests from datetime import datetime from typing import Any, Dict, List, Optional, Protocol import os +from eval_protocol.hosted_tracing_gateway import hosted_gateway_deprecation_message from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message from eval_protocol.tracing import PayloadType, decode_payloads from .base import BaseAdapter @@ -267,6 +269,10 @@ def get_final_generation_in_span_dict(trace: Dict[str, Any], span_name: str) -> class FireworksTracingAdapter(BaseAdapter): """Adapter to pull data from Langfuse via Fireworks tracing proxy. + DEPRECATED: The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on + October 28, 2026. This adapter depends on it and will be removed in a future release. Until then, + it only works with a self-hosted gateway that implements the same API, set via ``base_url``. + This adapter uses the Fireworks tracing proxy API which handles retry logic and rate limiting internally, simplifying the client-side implementation. @@ -305,6 +311,11 @@ def __init__( timeout: Request timeout in seconds (default: 300) api_key: Optional API key. If not provided, falls back to FIREWORKS_API_KEY environment variable. """ + warnings.warn( + hosted_gateway_deprecation_message("FireworksTracingAdapter", "base_url"), + DeprecationWarning, + stacklevel=2, + ) self.project_id = project_id self.base_url = base_url.rstrip("/") self.timeout = timeout diff --git a/eval_protocol/cli_commands/logs.py b/eval_protocol/cli_commands/logs.py index 63c48649..41592ab7 100644 --- a/eval_protocol/cli_commands/logs.py +++ b/eval_protocol/cli_commands/logs.py @@ -3,9 +3,11 @@ """ import sys +import warnings from pathlib import Path import os +from ..hosted_tracing_gateway import hosted_gateway_deprecation_message from ..utils.logs_server import serve_logs from ..event_bus.sqlite_event_bus_database import DatabaseCorruptedError, _backup_and_remove_database @@ -114,6 +116,14 @@ def logs_command(args): or os.environ.get("GATEWAY_URL") or "https://tracing.fireworks.ai" ) + if use_fireworks: + warnings.warn( + hosted_gateway_deprecation_message( + "The Fireworks backend of 'ep logs'", "FW_TRACING_GATEWAY_BASE_URL or GATEWAY_URL" + ), + DeprecationWarning, + stacklevel=2, + ) max_retries = 2 for attempt in range(max_retries): diff --git a/eval_protocol/hosted_tracing_gateway.py b/eval_protocol/hosted_tracing_gateway.py new file mode 100644 index 00000000..21db9661 --- /dev/null +++ b/eval_protocol/hosted_tracing_gateway.py @@ -0,0 +1,16 @@ +"""Deprecation notice for features built on the hosted Fireworks tracing gateway.""" + + +def hosted_gateway_deprecation_message(feature: str, configured_via: str) -> str: + """Build the deprecation warning for a feature that depends on the hosted tracing gateway. + + Args: + feature: User-facing name of the deprecated feature, e.g. "RemoteRolloutProcessor". + configured_via: Parameters or environment variables that point the feature at a self-hosted gateway. + """ + return ( + "The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. " + f"{feature} depends on it and is deprecated; it will be removed in a future release. " + "Until then, it only works with a self-hosted gateway that implements the same API, " + f"set via {configured_via}." + ) diff --git a/eval_protocol/log_utils/fireworks_tracing_http_handler.py b/eval_protocol/log_utils/fireworks_tracing_http_handler.py index 3ed52570..0082dc0b 100644 --- a/eval_protocol/log_utils/fireworks_tracing_http_handler.py +++ b/eval_protocol/log_utils/fireworks_tracing_http_handler.py @@ -1,16 +1,32 @@ import logging import os import threading +import warnings from datetime import datetime, timezone from typing import Optional, Any, Dict, List, cast import requests +from eval_protocol.hosted_tracing_gateway import hosted_gateway_deprecation_message + class FireworksTracingHttpHandler(logging.Handler): - """Logging handler that posts structured logs to tracing.fireworks gateway /logs endpoint.""" + """Logging handler that posts structured logs to tracing.fireworks gateway /logs endpoint. + + DEPRECATED: The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on + October 28, 2026. This handler depends on it and will be removed in a future release. Until then, + it only works with a self-hosted gateway that implements the same API, set via ``gateway_base_url`` + or ``FW_TRACING_GATEWAY_BASE_URL``. + """ def __init__(self, gateway_base_url: Optional[str] = None, rollout_id_env: str = "EP_ROLLOUT_ID") -> None: + warnings.warn( + hosted_gateway_deprecation_message( + "FireworksTracingHttpHandler", "gateway_base_url or FW_TRACING_GATEWAY_BASE_URL" + ), + DeprecationWarning, + stacklevel=2, + ) super().__init__() self.gateway_base_url = ( gateway_base_url or os.getenv("FW_TRACING_GATEWAY_BASE_URL") or "https://tracing.fireworks.ai" diff --git a/eval_protocol/log_utils/init.py b/eval_protocol/log_utils/init.py index 795cf7ba..ea970d22 100644 --- a/eval_protocol/log_utils/init.py +++ b/eval_protocol/log_utils/init.py @@ -39,11 +39,11 @@ def init_external_logging_from_env() -> None: # Ensure we do not add duplicate handlers if already present existing_handler_types = {type(h).__name__ for h in root_logger.handlers} - # Fireworks tracing: prefer if FIREWORKS_API_KEY is present; default base URL if not provided + # Fireworks tracing: only when FIREWORKS_API_KEY and a gateway URL are both set fw_key = _get_env("FIREWORKS_API_KEY") # Allow remote validation gateway to act as tracing base when provided - fw_url = _get_env("FW_TRACING_GATEWAY_BASE_URL") or _get_env("GATEWAY_URL") or "https://tracing.fireworks.ai" - if fw_key and "FireworksTracingHttpHandler" not in existing_handler_types: + fw_url = _get_env("FW_TRACING_GATEWAY_BASE_URL") or _get_env("GATEWAY_URL") + if fw_key and fw_url and "FireworksTracingHttpHandler" not in existing_handler_types: fw_handler = FireworksTracingHttpHandler(gateway_base_url=fw_url) fw_handler.setLevel(logging.INFO) fw_handler.addFilter(ContextRolloutIdFilter()) diff --git a/eval_protocol/pytest/github_action_rollout_processor.py b/eval_protocol/pytest/github_action_rollout_processor.py index 9e4cbb50..7096de06 100644 --- a/eval_protocol/pytest/github_action_rollout_processor.py +++ b/eval_protocol/pytest/github_action_rollout_processor.py @@ -1,10 +1,12 @@ import asyncio import os import time +import warnings from typing import Any, Dict, List, Optional import json import requests from datetime import datetime, timezone, timedelta +from eval_protocol.hosted_tracing_gateway import hosted_gateway_deprecation_message from eval_protocol.models import EvaluationRow, Status from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader @@ -24,6 +26,11 @@ class GithubActionRolloutProcessor(RolloutProcessor): - Traces are fetched later via Fireworks tracing proxy using rollout_id tags NOTE: GHA has a rate limit of 5000 requests per hour. + + DEPRECATED: The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on + October 28, 2026. This processor depends on it and will be removed in a future release. Until then, + it only works with a self-hosted gateway that implements the same API, set via ``model_base_url`` + or ``EP_MODEL_BASE_URL``. """ def __init__( @@ -39,6 +46,11 @@ def __init__( max_find_workflow_retries: int = 5, github_token: Optional[str] = None, ): + warnings.warn( + hosted_gateway_deprecation_message("GithubActionRolloutProcessor", "model_base_url or EP_MODEL_BASE_URL"), + DeprecationWarning, + stacklevel=2, + ) self.owner = owner self.repo = repo self.workflow_id = workflow_id diff --git a/eval_protocol/pytest/remote_rollout_processor.py b/eval_protocol/pytest/remote_rollout_processor.py index 05e49b38..b3453716 100644 --- a/eval_protocol/pytest/remote_rollout_processor.py +++ b/eval_protocol/pytest/remote_rollout_processor.py @@ -1,5 +1,6 @@ import asyncio import time +import warnings from typing import List, Optional import aiohttp @@ -7,6 +8,7 @@ from eval_protocol.models import EvaluationRow, Status from eval_protocol.adapters.fireworks_tracing import FireworksTracingAdapter from eval_protocol.exceptions import exception_for_status_code +from eval_protocol.hosted_tracing_gateway import hosted_gateway_deprecation_message from .rollout_processor import RolloutProcessor from .types import RolloutProcessorConfig @@ -25,6 +27,11 @@ class RemoteRolloutProcessor(RolloutProcessor): Fetches traces from the Fireworks tracing proxy using rollout_id tags. + DEPRECATED: The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on + October 28, 2026. This processor depends on it and will be removed in a future release. Until then, + it only works with a self-hosted gateway that implements the same API, set via ``model_base_url`` + or ``EP_MODEL_BASE_URL``. + See https://evalprotocol.io/tutorial/remote-rollout-processor for documentation. """ @@ -37,6 +44,11 @@ def __init__( timeout_seconds: float = 120.0, include_payloads: bool = False, ): + warnings.warn( + hosted_gateway_deprecation_message("RemoteRolloutProcessor", "model_base_url or EP_MODEL_BASE_URL"), + DeprecationWarning, + stacklevel=2, + ) # Prefer constructor-provided configuration. These can be overridden via # config.kwargs at call time for backward compatibility. self._remote_base_url = remote_base_url diff --git a/tests/test_hosted_tracing_gateway_deprecation.py b/tests/test_hosted_tracing_gateway_deprecation.py new file mode 100644 index 00000000..a45be14f --- /dev/null +++ b/tests/test_hosted_tracing_gateway_deprecation.py @@ -0,0 +1,124 @@ +"""Deprecation of the features built on the hosted Fireworks tracing gateway.""" + +import argparse +import logging +import warnings +from pathlib import Path +from typing import Callable, List + +import pytest + +import eval_protocol.cli_commands.logs as logs_cli +import eval_protocol.log_utils.init as log_init +from eval_protocol.adapters.fireworks_tracing import FireworksTracingAdapter +from eval_protocol.hosted_tracing_gateway import hosted_gateway_deprecation_message +from eval_protocol.log_utils.fireworks_tracing_http_handler import FireworksTracingHttpHandler +from eval_protocol.pytest.github_action_rollout_processor import GithubActionRolloutProcessor +from eval_protocol.pytest.remote_rollout_processor import RemoteRolloutProcessor + +SHUTDOWN_NOTICE = ( + "The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026." +) +SELF_HOSTED_GATEWAY_URL = "http://self-hosted-gateway.test" + + +@pytest.fixture(autouse=True) +def clear_gateway_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("FIREWORKS_API_KEY", "FW_TRACING_GATEWAY_BASE_URL", "GATEWAY_URL"): + monkeypatch.delenv(name, raising=False) + + +def _gateway_deprecations(caught: List[warnings.WarningMessage]) -> List[warnings.WarningMessage]: + return [w for w in caught if issubclass(w.category, DeprecationWarning) and SHUTDOWN_NOTICE in str(w.message)] + + +def _attach_external_logging() -> List[logging.Handler]: + """Run init_external_logging_from_env and detach whatever it attached to the root logger.""" + root = logging.getLogger() + before = list(root.handlers) + log_init.init_external_logging_from_env() + added = [handler for handler in root.handlers if handler not in before] + for handler in added: + root.removeHandler(handler) + handler.close() + return added + + +def test_deprecation_message_wording() -> None: + assert hosted_gateway_deprecation_message("Feature", "feature_url") == ( + "The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. " + "Feature depends on it and is deprecated; it will be removed in a future release. " + "Until then, it only works with a self-hosted gateway that implements the same API, set via feature_url." + ) + + +@pytest.mark.parametrize( + ("construct", "feature"), + [ + (lambda: RemoteRolloutProcessor(remote_base_url="http://127.0.0.1:1"), "RemoteRolloutProcessor"), + ( + lambda: GithubActionRolloutProcessor(owner="owner", repo="repo", workflow_id="rollout.yml"), + "GithubActionRolloutProcessor", + ), + (lambda: FireworksTracingAdapter(), "FireworksTracingAdapter"), + (lambda: FireworksTracingHttpHandler(), "FireworksTracingHttpHandler"), + ], +) +def test_entry_point_warns_at_caller(construct: Callable[[], object], feature: str) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + construct() + + deprecations = [w for w in _gateway_deprecations(caught) if f"{feature} depends on it" in str(w.message)] + assert len(deprecations) == 1 + assert Path(deprecations[0].filename) == Path(__file__) + + +def test_init_does_not_attach_fireworks_handler_without_gateway_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FIREWORKS_API_KEY", "test-key") + monkeypatch.setattr(log_init, "_INITIALIZED", False) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + added = _attach_external_logging() + + assert added == [] + assert _gateway_deprecations(caught) == [] + + +@pytest.mark.parametrize("gateway_env_var", ["FW_TRACING_GATEWAY_BASE_URL", "GATEWAY_URL"]) +def test_init_attaches_fireworks_handler_for_explicit_gateway_url( + monkeypatch: pytest.MonkeyPatch, gateway_env_var: str +) -> None: + monkeypatch.setenv("FIREWORKS_API_KEY", "test-key") + monkeypatch.setenv(gateway_env_var, SELF_HOSTED_GATEWAY_URL) + monkeypatch.setattr(log_init, "_INITIALIZED", False) + + with pytest.warns(DeprecationWarning, match="FireworksTracingHttpHandler depends on it"): + added = _attach_external_logging() + + assert len(added) == 1 + assert isinstance(added[0], log_init.FireworksTracingHttpHandler) + assert added[0].gateway_base_url == SELF_HOSTED_GATEWAY_URL + + +def _logs_args() -> argparse.Namespace: + return argparse.Namespace(port=8000, debug=False, use_fireworks=False, use_elasticsearch=False) + + +def test_ep_logs_warns_when_using_fireworks_backend(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FIREWORKS_API_KEY", "test-key") + monkeypatch.setattr(logs_cli, "serve_logs", lambda **kwargs: None) + + with pytest.warns(DeprecationWarning, match="The Fireworks backend of 'ep logs' depends on it"): + assert logs_cli.logs_command(_logs_args()) == 0 + + +def test_ep_logs_does_not_warn_for_elasticsearch_backend(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(logs_cli, "serve_logs", lambda **kwargs: None) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert logs_cli.logs_command(_logs_args()) == 0 + + assert _gateway_deprecations(caught) == [] diff --git a/typescript/README.md b/typescript/README.md index 03727695..2d9405e2 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -6,6 +6,8 @@ It includes helpful Zod schemas. TypeScript types, and helper functions for creating a remote server for [RemoteRolloutProcessor](https://evalprotocol.io/tutorial/remote-rollout-processor). +> **Deprecated:** The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. `RemoteRolloutProcessor` and the logging helpers in this package (`FireworksTransport`, `logger`, `createRolloutLogger`) depend on it and will be removed in a future release. + Installation ```bash diff --git a/typescript/logging/fireworks-transport.ts b/typescript/logging/fireworks-transport.ts index 90581115..ccc38464 100644 --- a/typescript/logging/fireworks-transport.ts +++ b/typescript/logging/fireworks-transport.ts @@ -5,6 +5,7 @@ import Transport from 'winston-transport'; import type { TransformableInfo } from 'logform'; const LEVEL = Symbol.for('level'); +let warnedAboutHostedGateway = false; interface FireworksLogInfo extends TransformableInfo { rollout_id?: string; @@ -37,6 +38,12 @@ interface FireworksPayload { }; } +/** + * @deprecated The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on + * October 28, 2026. FireworksTransport depends on it and will be removed in a future release. Until then, + * it only works with a self-hosted gateway that implements the same API, set via `gatewayBaseUrl` or + * `FW_TRACING_GATEWAY_BASE_URL`. + */ export class FireworksTransport extends Transport { private gatewayBaseUrl: string; private rolloutIdEnv: string; @@ -92,6 +99,10 @@ export class FireworksTransport extends Transport { const payload = this.buildPayload(info, rolloutId); const baseUrl = this.gatewayBaseUrl.replace(/\/$/, ''); const url = `${baseUrl}/logs`; + if (baseUrl === 'https://tracing.fireworks.ai' && !warnedAboutHostedGateway) { + warnedAboutHostedGateway = true; + console.warn('[FW_LOG] The hosted Fireworks tracing gateway (https://tracing.fireworks.ai) was shut down on October 28, 2026. FireworksTransport depends on it and is deprecated; it will be removed in a future release.'); + } let payloadJson: string; try { diff --git a/typescript/logging/logger.ts b/typescript/logging/logger.ts index 33bb4a59..53fcc5f2 100644 --- a/typescript/logging/logger.ts +++ b/typescript/logging/logger.ts @@ -19,6 +19,10 @@ export function setWaitUntil(waitUntil: (promise: Promise) => void) { globalWaitUntil = waitUntil; } +/** + * @deprecated Forwards logs through FireworksTransport to the hosted Fireworks tracing gateway + * (https://tracing.fireworks.ai), which was shut down on October 28, 2026. Will be removed in a future release. + */ export const logger = winston.createLogger({ level: 'info', format: winston.format.combine( @@ -41,6 +45,9 @@ export const logger = winston.createLogger({ /** * Create a child logger with rollout_id context. + * + * @deprecated Forwards logs through FireworksTransport to the hosted Fireworks tracing gateway + * (https://tracing.fireworks.ai), which was shut down on October 28, 2026. Will be removed in a future release. */ export function createRolloutLogger( rolloutId: string,