diff --git a/Cargo.lock b/Cargo.lock index 278c86f3e..915664380 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1797,6 +1797,7 @@ dependencies = [ "futures-util", "http", "http-body-util", + "httpdate", "hyper", "hyper-rustls", "hyper-util", diff --git a/crates/adaptive/src/acg/request_surfaces/mod.rs b/crates/adaptive/src/acg/request_surfaces/mod.rs index b0e3ec4eb..5c8b7f5f6 100644 --- a/crates/adaptive/src/acg/request_surfaces/mod.rs +++ b/crates/adaptive/src/acg/request_surfaces/mod.rs @@ -52,6 +52,8 @@ impl RequestSurface { ProviderSurface::OCIGenAI => None, // Gemini generateContent ACG request editing is intentionally unsupported. ProviderSurface::GeminiGenerateContent => None, + // ACG prompt mutation is not defined for evaluation state/questions. + ProviderSurface::TypeSafeSystemOne => None, } } diff --git a/crates/adaptive/src/response_cache/key.rs b/crates/adaptive/src/response_cache/key.rs index 645a5c112..586d05705 100644 --- a/crates/adaptive/src/response_cache/key.rs +++ b/crates/adaptive/src/response_cache/key.rs @@ -428,6 +428,9 @@ fn lossy_request_shape(surface: ProviderSurface, content: &Json) -> bool { .and_then(Json::as_array) .is_some_and(|items| items.iter().any(lossy_gemini_content_item)) } + // The codec losslessly preserves evaluation state, questions, and + // provider extensions, so normalized cache keys are safe. + ProviderSurface::TypeSafeSystemOne => false, } } diff --git a/crates/adaptive/src/response_cache/replay.rs b/crates/adaptive/src/response_cache/replay.rs index 43d7fde1d..edfa68435 100644 --- a/crates/adaptive/src/response_cache/replay.rs +++ b/crates/adaptive/src/response_cache/replay.rs @@ -90,6 +90,8 @@ fn synthesize_replay_chunks(aggregate: &Json) -> Option> { // re-aggregates it and rejects shapes the streaming collector cannot // preserve exactly. ProviderSurface::GeminiGenerateContent => vec![aggregate.clone()], + // System One is explicitly non-streaming, so no replay chunks exist. + ProviderSurface::TypeSafeSystemOne => return None, }) } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 0851bbb06..d4cb1e5d2 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -42,6 +42,7 @@ futures-util = "0.3" fs2 = "0.4" http = "1" http-body-util = "0.1" +httpdate = "1" hyper = { version = "1.11.1", features = ["client", "server", "http1", "http2"] } hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] } hyper-util = { version = "0.1", features = ["client-legacy", "http1", "http2", "server-auto", "service", "tokio"] } diff --git a/crates/cli/src/agents/shared/alignment.rs b/crates/cli/src/agents/shared/alignment.rs index 834045361..4128f9be5 100644 --- a/crates/cli/src/agents/shared/alignment.rs +++ b/crates/cli/src/agents/shared/alignment.rs @@ -51,16 +51,20 @@ pub(crate) enum GatewayRouteKind { OpenAiModels, AnthropicMessages, AnthropicCountTokens, + TypeSafeSystemOne, + TypeSafeModels, } impl GatewayRouteKind { - pub(crate) const ALL: [Self; 6] = [ + pub(crate) const ALL: [Self; 8] = [ Self::OpenAiResponses, Self::OpenAiChatCompletions, Self::OpenAiImagesGenerations, Self::OpenAiModels, Self::AnthropicMessages, Self::AnthropicCountTokens, + Self::TypeSafeSystemOne, + Self::TypeSafeModels, ]; pub(crate) const fn name(self) -> &'static str { @@ -71,6 +75,8 @@ impl GatewayRouteKind { Self::OpenAiModels => "openai.models", Self::AnthropicMessages => "anthropic.messages", Self::AnthropicCountTokens => "anthropic.count_tokens", + Self::TypeSafeSystemOne => "typesafe.system_one", + Self::TypeSafeModels => "typesafe.models", } } @@ -495,6 +501,8 @@ fn provider_request_extractor(route: GatewayRouteKind) -> &'static dyn ProviderR GatewayRouteKind::OpenAiModels => &OPENAI_MODELS_REQUEST_EXTRACTOR, GatewayRouteKind::AnthropicMessages => &ANTHROPIC_MESSAGES_REQUEST_EXTRACTOR, GatewayRouteKind::AnthropicCountTokens => &ANTHROPIC_COUNT_TOKENS_REQUEST_EXTRACTOR, + GatewayRouteKind::TypeSafeSystemOne => &OPENAI_MODELS_REQUEST_EXTRACTOR, + GatewayRouteKind::TypeSafeModels => &OPENAI_MODELS_REQUEST_EXTRACTOR, } } diff --git a/crates/cli/src/commands/configure/editor.rs b/crates/cli/src/commands/configure/editor.rs index 26560356a..82b5381b4 100644 --- a/crates/cli/src/commands/configure/editor.rs +++ b/crates/cli/src/commands/configure/editor.rs @@ -105,15 +105,23 @@ impl ConfigDocument { } fn has_auth_headers(&self) -> bool { - ["openai_auth_header", "anthropic_auth_header"] - .into_iter() - .any(|key| self.has_key("upstream", key)) + [ + "openai_auth_header", + "anthropic_auth_header", + "typesafe_auth_header", + ] + .into_iter() + .any(|key| self.has_key("upstream", key)) } fn preview(&self) -> String { let mut document = self.document.clone(); if let Some(upstream) = document.get_mut("upstream") { - for key in ["openai_auth_header", "anthropic_auth_header"] { + for key in [ + "openai_auth_header", + "anthropic_auth_header", + "typesafe_auth_header", + ] { if let Some(table) = upstream.as_table_mut() { if table.contains_key(key) { table[key] = value(""); diff --git a/crates/cli/src/commands/configure/editor/prompt.rs b/crates/cli/src/commands/configure/editor/prompt.rs index bb2f62465..8de7878cc 100644 --- a/crates/cli/src/commands/configure/editor/prompt.rs +++ b/crates/cli/src/commands/configure/editor/prompt.rs @@ -126,6 +126,14 @@ fn edit_upstream(theme: &ColorfulTheme, document: &mut ConfigDocument) -> Result "Anthropic authorization header: {}", document.secret_summary("anthropic_auth_header") ), + format!( + "TypeSafe base URL: {}", + document.string_summary("upstream", "typesafe_base_url") + ), + format!( + "TypeSafe authorization header: {}", + document.secret_summary("typesafe_auth_header") + ), "Back".into(), ]; match select(theme, "Provider upstreams", &choices)? { @@ -133,7 +141,9 @@ fn edit_upstream(theme: &ColorfulTheme, document: &mut ConfigDocument) -> Result 1 => edit_secret(theme, document, "openai_auth_header")?, 2 => edit_string(theme, document, "upstream", "anthropic_base_url")?, 3 => edit_secret(theme, document, "anthropic_auth_header")?, - 4 => return Ok(()), + 4 => edit_string(theme, document, "upstream", "typesafe_base_url")?, + 5 => edit_secret(theme, document, "typesafe_auth_header")?, + 6 => return Ok(()), _ => unreachable!(), } } diff --git a/crates/cli/src/commands/run.rs b/crates/cli/src/commands/run.rs index d1dcc2ba6..fc9d9bbac 100644 --- a/crates/cli/src/commands/run.rs +++ b/crates/cli/src/commands/run.rs @@ -32,6 +32,8 @@ pub(crate) struct RunCommand { #[arg(long)] pub(super) anthropic_base_url: Option, #[arg(long)] + pub(super) typesafe_base_url: Option, + #[arg(long)] pub(super) session_metadata: Option, #[arg(long, env = "NEMO_RELAY_PLUGIN_CONFIG_PATH", hide = true)] pub(super) plugin_config_path: Option, @@ -50,6 +52,7 @@ impl RunCommand { config: self.config, openai_base_url: self.openai_base_url, anthropic_base_url: self.anthropic_base_url, + typesafe_base_url: self.typesafe_base_url, session_metadata: self.session_metadata, plugin_config_path: self.plugin_config_path, dry_run: self.dry_run, @@ -106,6 +109,7 @@ pub(super) async fn easy_path( config: explicit_config.map(PathBuf::from), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: command.dry_run, diff --git a/crates/cli/src/commands/serve.rs b/crates/cli/src/commands/serve.rs index e0d99e255..9b1e855d1 100644 --- a/crates/cli/src/commands/serve.rs +++ b/crates/cli/src/commands/serve.rs @@ -20,6 +20,9 @@ pub(crate) struct ServerArgs { /// Upstream Anthropic base URL (e.g. https://api.anthropic.com) #[arg(long, env = "NEMO_RELAY_ANTHROPIC_BASE_URL")] pub(super) anthropic_base_url: Option, + /// Upstream TypeSafe base URL (e.g. https://api.typesafe.ai/v1) + #[arg(long, env = "NEMO_RELAY_TYPESAFE_BASE_URL")] + pub(super) typesafe_base_url: Option, /// Internal override for the plugin configuration file. #[arg(long, env = "NEMO_RELAY_PLUGIN_CONFIG_PATH", hide = true)] pub(super) plugin_config_path: Option, @@ -41,6 +44,7 @@ impl ServerArgs { bind: self.bind, openai_base_url: self.openai_base_url.clone(), anthropic_base_url: self.anthropic_base_url.clone(), + typesafe_base_url: self.typesafe_base_url.clone(), plugin_config_path: self.plugin_config_path.clone(), ready_file: self.ready_file.clone(), max_hook_payload_bytes: self.max_hook_payload_bytes, diff --git a/crates/cli/src/configuration/mod.rs b/crates/cli/src/configuration/mod.rs index 8b0e06995..7019130a5 100644 --- a/crates/cli/src/configuration/mod.rs +++ b/crates/cli/src/configuration/mod.rs @@ -75,6 +75,10 @@ struct FileUpstreamConfig { openai_auth_header: Option, anthropic_base_url: Option, anthropic_auth_header: Option, + typesafe_base_url: Option, + typesafe_auth_header: Option, + typesafe_max_retries: Option, + typesafe_max_retry_delay_ms: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -297,6 +301,10 @@ fn persistent_bootstrap_fingerprint( "openai_auth_header": gateway.openai_auth_header, "anthropic_base_url": gateway.anthropic_base_url, "anthropic_auth_header": gateway.anthropic_auth_header, + "typesafe_base_url": gateway.typesafe_base_url, + "typesafe_auth_header": gateway.typesafe_auth_header, + "typesafe_max_retries": gateway.typesafe_retry.max_retries, + "typesafe_max_retry_delay_ms": gateway.typesafe_retry.max_server_delay.as_millis(), "metadata": gateway.metadata, "plugin_config": gateway.plugin_config, "max_hook_payload_bytes": gateway.max_hook_payload_bytes, @@ -1101,6 +1109,13 @@ fn apply_run_url_overrides(config: &mut GatewayConfig, command: &RunOverrides) { value.clone(), ); } + if let Some(value) = &command.typesafe_base_url { + replace_upstream_base_url( + &mut config.typesafe_base_url, + &mut config.typesafe_auth_header, + value.clone(), + ); + } } // Parses JSON-bearing run overrides after simple values. Invalid metadata or plugin config fails @@ -1138,6 +1153,13 @@ fn apply_server_overrides( value.clone(), ); } + if let Some(value) = &args.typesafe_base_url { + replace_upstream_base_url( + &mut config.typesafe_base_url, + &mut config.typesafe_auth_header, + value.clone(), + ); + } if let Some(value) = args.max_hook_payload_bytes { config.max_hook_payload_bytes = validate_body_limit("max hook payload bytes", value)?; } @@ -1397,6 +1419,10 @@ fn apply_file_upstream_config( openai_auth_header, anthropic_base_url, anthropic_auth_header, + typesafe_base_url, + typesafe_auth_header, + typesafe_max_retries, + typesafe_max_retry_delay_ms, } = upstream; if let Some(value) = openai_base_url { gateway.openai_base_url = value; @@ -1420,6 +1446,24 @@ fn apply_file_upstream_config( value, )?); } + if let Some(value) = typesafe_base_url { + gateway.typesafe_base_url = value; + if typesafe_auth_header.is_none() { + gateway.typesafe_auth_header = None; + } + } + if let Some(value) = typesafe_auth_header { + gateway.typesafe_auth_header = Some(validate_auth_header( + "upstream.typesafe_auth_header", + value, + )?); + } + if let Some(value) = typesafe_max_retries { + gateway.typesafe_retry.max_retries = value; + } + if let Some(value) = typesafe_max_retry_delay_ms { + gateway.typesafe_retry.max_server_delay = Duration::from_millis(value); + } Ok(()) } @@ -1724,6 +1768,30 @@ fn apply_env_config(config: &mut GatewayConfig) -> Result<(), CliError> { value, )?); } + let typesafe_auth_header = std::env::var("NEMO_RELAY_TYPESAFE_AUTH_HEADER").ok(); + if let Ok(value) = std::env::var("NEMO_RELAY_TYPESAFE_BASE_URL") { + replace_upstream_base_url( + &mut config.typesafe_base_url, + &mut config.typesafe_auth_header, + value, + ); + } + if let Some(value) = typesafe_auth_header { + config.typesafe_auth_header = Some(validate_auth_header( + "NEMO_RELAY_TYPESAFE_AUTH_HEADER", + value, + )?); + } + if let Ok(value) = std::env::var("NEMO_RELAY_TYPESAFE_MAX_RETRIES") { + config.typesafe_retry.max_retries = + parse_retry_u32("NEMO_RELAY_TYPESAFE_MAX_RETRIES", &value)?; + } + if let Ok(value) = std::env::var("NEMO_RELAY_TYPESAFE_MAX_RETRY_DELAY_MS") { + config.typesafe_retry.max_server_delay = Duration::from_millis(parse_retry_u64( + "NEMO_RELAY_TYPESAFE_MAX_RETRY_DELAY_MS", + &value, + )?); + } if let Ok(value) = std::env::var("NEMO_RELAY_MAX_HOOK_PAYLOAD_BYTES") { config.max_hook_payload_bytes = parse_env_body_limit("NEMO_RELAY_MAX_HOOK_PAYLOAD_BYTES", &value)?; @@ -1751,6 +1819,12 @@ fn apply_managed_worker_env_config(config: &mut GatewayConfig) -> Result<(), Cli value, )?); } + if let Ok(value) = std::env::var("NEMO_RELAY_TYPESAFE_AUTH_HEADER") { + config.typesafe_auth_header = Some(validate_auth_header( + "NEMO_RELAY_TYPESAFE_AUTH_HEADER", + value, + )?); + } Ok(()) } @@ -1782,6 +1856,18 @@ fn parse_env_body_limit(name: &str, raw: &str) -> Result { validate_body_limit(name, value) } +fn parse_retry_u32(name: &str, raw: &str) -> Result { + raw.trim() + .parse::() + .map_err(|_| CliError::Config(format!("{name} must be a non-negative integer"))) +} + +fn parse_retry_u64(name: &str, raw: &str) -> Result { + raw.trim() + .parse::() + .map_err(|_| CliError::Config(format!("{name} must be a non-negative integer"))) +} + fn validate_body_limit(name: &str, value: usize) -> Result { if value == 0 { return Err(CliError::Config(format!("{name} must be greater than 0"))); @@ -1824,6 +1910,7 @@ fn clear_credentials_for_replaced_upstreams(left: &mut toml::Value, right: &toml for (base_url, auth_header) in [ ("openai_base_url", "openai_auth_header"), ("anthropic_base_url", "anthropic_auth_header"), + ("typesafe_base_url", "typesafe_auth_header"), ] { let endpoint_changed = override_upstream .get(base_url) diff --git a/crates/cli/src/configuration/types.rs b/crates/cli/src/configuration/types.rs index 851bde012..623b3693b 100644 --- a/crates/cli/src/configuration/types.rs +++ b/crates/cli/src/configuration/types.rs @@ -25,6 +25,9 @@ pub(crate) struct GatewayConfig { pub(crate) openai_auth_header: Option, pub(crate) anthropic_base_url: String, pub(crate) anthropic_auth_header: Option, + pub(crate) typesafe_base_url: String, + pub(crate) typesafe_auth_header: Option, + pub(crate) typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy, pub(crate) metadata: Option, pub(crate) plugin_config: Option, pub(crate) max_hook_payload_bytes: usize, @@ -115,6 +118,9 @@ impl Default for GatewayConfig { openai_auth_header: None, anthropic_base_url: "https://api.anthropic.com".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: DEFAULT_MAX_HOOK_PAYLOAD_BYTES, diff --git a/crates/cli/src/daemon/broker/server.rs b/crates/cli/src/daemon/broker/server.rs index a4fc1e3cd..9fa679c45 100644 --- a/crates/cli/src/daemon/broker/server.rs +++ b/crates/cli/src/daemon/broker/server.rs @@ -1337,8 +1337,11 @@ async fn probe_worker(target: &Arc) -> Result<(), CliError> { } async fn public_proxy(state: State>, request: Request) -> Response { - let catalog = - request.method() == Method::GET && matches!(request.uri().path(), "/models" | "/v1/models"); + let catalog = request.method() == Method::GET + && matches!( + request.uri().path(), + "/models" | "/v1/models" | "/typesafe/models" | "/typesafe/v1/models" + ); let mut response = public_proxy_inner(state, request).await; if catalog { // Shared catalog URLs are keyed by a private credential, not by their URI. Override @@ -1410,7 +1413,10 @@ fn responses_websocket_probe(request: &Request) -> bool { } fn public_method_allowed(method: &Method, path: &str) -> bool { - if matches!(path, "/models" | "/v1/models") { + if matches!( + path, + "/models" | "/v1/models" | "/typesafe/models" | "/typesafe/v1/models" + ) { method == Method::GET } else { method == Method::POST @@ -1491,6 +1497,7 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & let configured = match route { ProviderRoute::OpenAi => config.openai_auth_header.as_deref(), ProviderRoute::Anthropic => config.anthropic_auth_header.as_deref(), + ProviderRoute::TypeSafe => config.typesafe_auth_header.as_deref(), }; if let Some(configured) = configured.and_then(|value| HeaderValue::from_str(value).ok()) { headers.insert(AUTHORIZATION, configured); @@ -1509,6 +1516,12 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & }; (HeaderName::from_static("x-api-key"), key) } + ProviderRoute::TypeSafe => { + let Some(key) = nonempty_environment("TYPESAFE_API_KEY") else { + return; + }; + (AUTHORIZATION, format!("Bearer {key}")) + } }; if let Ok(value) = HeaderValue::from_str(&value) { headers.insert(name, value); diff --git a/crates/cli/src/daemon/common/routes.rs b/crates/cli/src/daemon/common/routes.rs index 3ebe26bea..a573476e3 100644 --- a/crates/cli/src/daemon/common/routes.rs +++ b/crates/cli/src/daemon/common/routes.rs @@ -22,6 +22,7 @@ pub(crate) enum HookRoute { pub(crate) enum ProviderRoute { OpenAi, Anthropic, + TypeSafe, } impl PublicRoute { @@ -41,6 +42,12 @@ impl PublicRoute { "/v1/messages" | "/v1/messages/count_tokens" => { Some(Self::Provider(ProviderRoute::Anthropic)) } + "/systemone" + | "/v1/systemone" + | "/typesafe/systemone" + | "/typesafe/v1/systemone" + | "/typesafe/models" + | "/typesafe/v1/models" => Some(Self::Provider(ProviderRoute::TypeSafe)), _ => None, } } @@ -60,6 +67,7 @@ impl ProviderRoute { match self { Self::OpenAi => "openai", Self::Anthropic => "anthropic", + Self::TypeSafe => "typesafe", } } @@ -67,11 +75,16 @@ impl ProviderRoute { let base = match self { Self::OpenAi => config.openai_base_url.as_str(), Self::Anthropic => config.anthropic_base_url.as_str(), + Self::TypeSafe => config.typesafe_base_url.as_str(), } .trim_end_matches('/'); let path = match self { Self::OpenAi => canonical_openai_path(path_and_query), Self::Anthropic => path_and_query.to_owned(), + Self::TypeSafe => path_and_query + .strip_prefix("/typesafe") + .unwrap_or(path_and_query) + .to_owned(), }; let path = normalize_v1_path(base, &path); format!("{base}{path}") diff --git a/crates/cli/src/daemon/worker/managed.rs b/crates/cli/src/daemon/worker/managed.rs index 5188b2b61..25b10cd6a 100644 --- a/crates/cli/src/daemon/worker/managed.rs +++ b/crates/cli/src/daemon/worker/managed.rs @@ -376,7 +376,9 @@ impl ManagedRuntime { return dispatch_unmanaged(upstream, request, route, &self.config).await; }; let operational = OperationalContext::take_from_headers(request.headers_mut()); - if !middleware.request_body_decode_required { + // TypeSafe requests are replayable JSON and its SDK contract includes retries. Buffer them + // even when no plugin needs request decoding so daemon and direct-gateway behavior match. + if !middleware.request_body_decode_required && route != ProviderRoute::TypeSafe { strip_worker_headers(request.headers_mut()); strip_untrusted_dispatch_headers(request.headers_mut()); let streaming_hint = request_streaming_hint(request.headers()); @@ -721,13 +723,65 @@ async fn dispatch_unmanaged( HeaderName::from_static(WORKER_TOKEN_HEADER), HeaderName::from_static(WORKER_ROUTE_FAILURE_HEADER), ]; - let request = prepare_forward_request(request, destination, &strip) - .map_err(|error| CliError::InvalidPayload(error.to_string()))? - .map(box_body); - let response = tokio::time::timeout(RESPONSE_HEAD_TIMEOUT, upstream.request(request)) - .await - .map_err(|_| CliError::Launch("provider response-head timeout".into()))? - .map_err(|error| CliError::Launch(error.to_string()))?; + // The only unmanaged TypeSafe route is the read-only model catalog. Rebuild its empty GET + // request for retries without aggregating a body in this streaming forwarding layer. + let response = if route == ProviderRoute::TypeSafe && request.method() == Method::GET { + let (parts, _body) = request.into_parts(); + let mut retry_count = 0_u32; + loop { + let mut replay = Request::builder() + .method(parts.method.clone()) + .version(parts.version) + .uri(destination.clone()) + .body(Body::empty())?; + *replay.headers_mut() = parts.headers.clone(); + replay.headers_mut().remove("x-typesafe-retry-count"); + if retry_count > 0 { + replay.headers_mut().insert( + HeaderName::from_static("x-typesafe-retry-count"), + HeaderValue::from_str(&retry_count.to_string()) + .expect("retry count is a valid header value"), + ); + } + let replay = prepare_forward_request(replay, destination.clone(), &strip) + .map_err(|error| CliError::InvalidPayload(error.to_string()))? + .map(box_body); + let attempt = + tokio::time::timeout(RESPONSE_HEAD_TIMEOUT, upstream.request(replay)).await; + let response = match attempt { + Ok(Ok(response)) => response, + Ok(Err(_)) | Err(_) if config.typesafe_retry.can_retry_transport(retry_count) => { + let delay = config.typesafe_retry.delay(None, retry_count); + retry_count += 1; + tokio::time::sleep(delay).await; + continue; + } + Ok(Err(error)) => return Err(CliError::Launch(error.to_string())), + Err(_) => return Err(CliError::Launch("provider response-head timeout".into())), + }; + if config + .typesafe_retry + .should_retry_status(response.status(), retry_count) + { + let delay = config + .typesafe_retry + .delay(Some(response.headers()), retry_count); + retry_count += 1; + drop(response); + tokio::time::sleep(delay).await; + continue; + } + break response; + } + } else { + let request = prepare_forward_request(request, destination, &strip) + .map_err(|error| CliError::InvalidPayload(error.to_string()))? + .map(box_body); + tokio::time::timeout(RESPONSE_HEAD_TIMEOUT, upstream.request(request)) + .await + .map_err(|_| CliError::Launch("provider response-head timeout".into()))? + .map_err(|error| CliError::Launch(error.to_string()))? + }; let response = prepare_forward_response(response, &strip) .map_err(|error| CliError::Launch(error.to_string()))?; let (parts, body) = response.into_parts(); @@ -864,25 +918,83 @@ async fn dispatch_observed( { headers = aligned; } - let mut request = Request::builder() - .method(prepared.method.clone()) - .version(prepared.version) - .uri(destination.clone()) - .body(Body::from(body))?; - *request.headers_mut() = headers; - if !explicit_target && allow_environment_provider_auth { - inject_provider_auth(request.headers_mut(), route, config); - } let strip = [ HeaderName::from_static(CLIENT_TOKEN_HEADER), HeaderName::from_static(WORKER_TOKEN_HEADER), HeaderName::from_static(WORKER_ROUTE_FAILURE_HEADER), ]; - let request = prepare_forward_request(request, destination, &strip) - .map_err(|error| CliError::InvalidPayload(error.to_string()))? - .map(box_body); - let response = - request_worker_upstream(upstream, request, &operational, prepared.streaming).await?; + let mut retry_count = 0_u32; + let response = loop { + let mut request = Request::builder() + .method(prepared.method.clone()) + .version(prepared.version) + .uri(destination.clone()) + .body(Body::from(body.clone()))?; + *request.headers_mut() = headers.clone(); + request.headers_mut().remove("x-typesafe-retry-count"); + if route == ProviderRoute::TypeSafe && retry_count > 0 { + request.headers_mut().insert( + HeaderName::from_static("x-typesafe-retry-count"), + HeaderValue::from_str(&retry_count.to_string()) + .expect("retry count is a valid header value"), + ); + } + if !explicit_target && allow_environment_provider_auth { + inject_provider_auth(request.headers_mut(), route, config); + } + let request = prepare_forward_request(request, destination.clone(), &strip) + .map_err(|error| CliError::InvalidPayload(error.to_string()))? + .map(box_body); + let attempt = + request_worker_upstream(upstream.clone(), request, &operational, prepared.streaming) + .await; + let response = match attempt { + Ok(response) => response, + Err(_error) + if route == ProviderRoute::TypeSafe + && config.typesafe_retry.can_retry_transport(retry_count) => + { + let delay = config.typesafe_retry.delay(None, retry_count); + retry_count += 1; + operational::upstream_retry_scheduled( + &operational, + "typesafe", + "transport", + retry_count, + delay.as_millis().try_into().unwrap_or(u64::MAX), + ); + tokio::time::sleep(delay).await; + continue; + } + Err(error) => return Err(error), + }; + if route == ProviderRoute::TypeSafe + && config + .typesafe_retry + .should_retry_status(response.status(), retry_count) + { + let status = response.status(); + let delay = config + .typesafe_retry + .delay(Some(response.headers()), retry_count); + retry_count += 1; + operational::upstream_retry_scheduled( + &operational, + "typesafe", + if status == StatusCode::TOO_MANY_REQUESTS { + "rate_limit" + } else { + "status" + }, + retry_count, + delay.as_millis().try_into().unwrap_or(u64::MAX), + ); + drop(response); + tokio::time::sleep(delay).await; + continue; + } + break response; + }; let response = prepare_forward_response(response, &strip).map_err(|error| { operational::upstream_failed(&operational, "invalid_response"); CliError::Launch(error.to_string()) @@ -1074,6 +1186,7 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & if let Some(configured) = match route { ProviderRoute::OpenAi => config.openai_auth_header.as_deref(), ProviderRoute::Anthropic => config.anthropic_auth_header.as_deref(), + ProviderRoute::TypeSafe => config.typesafe_auth_header.as_deref(), } .and_then(|value| HeaderValue::from_str(value).ok()) { @@ -1093,6 +1206,12 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & }; (HeaderName::from_static("x-api-key"), key) } + ProviderRoute::TypeSafe => { + let Some(key) = environment_value("TYPESAFE_API_KEY") else { + return; + }; + (AUTHORIZATION, format!("Bearer {key}")) + } }; if let Ok(value) = HeaderValue::from_str(&value) { headers.insert(name, value); @@ -1143,6 +1262,9 @@ fn provider_surface(path: &str) -> Option { } "/chat/completions" | "/v1/chat/completions" => Some(ProviderSurface::OpenAIChat), "/v1/messages" => Some(ProviderSurface::AnthropicMessages), + "/systemone" | "/v1/systemone" | "/typesafe/systemone" | "/typesafe/v1/systemone" => { + Some(ProviderSurface::TypeSafeSystemOne) + } _ => None, } } diff --git a/crates/cli/src/daemon/worker/runtime.rs b/crates/cli/src/daemon/worker/runtime.rs index 09e2956a2..431d82385 100644 --- a/crates/cli/src/daemon/worker/runtime.rs +++ b/crates/cli/src/daemon/worker/runtime.rs @@ -799,6 +799,7 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & let configured = match route { ProviderRoute::OpenAi => config.openai_auth_header.as_deref(), ProviderRoute::Anthropic => config.anthropic_auth_header.as_deref(), + ProviderRoute::TypeSafe => config.typesafe_auth_header.as_deref(), }; if let Some(configured) = configured.and_then(header_value) { headers.insert(AUTHORIZATION, configured); @@ -821,6 +822,14 @@ fn inject_provider_auth(headers: &mut HeaderMap, route: ProviderRoute, config: & headers.insert(HeaderName::from_static("x-api-key"), value); } } + ProviderRoute::TypeSafe => { + let Some(key) = nonempty_environment("TYPESAFE_API_KEY") else { + return; + }; + if let Some(value) = header_value(&format!("Bearer {key}")) { + headers.insert(AUTHORIZATION, value); + } + } } } diff --git a/crates/cli/src/diagnostics/model.rs b/crates/cli/src/diagnostics/model.rs index 993948f2a..ee872fb55 100644 --- a/crates/cli/src/diagnostics/model.rs +++ b/crates/cli/src/diagnostics/model.rs @@ -105,6 +105,7 @@ pub(crate) struct PluginHostValidation { pub(crate) struct UpstreamAuthInfo { pub openai: SecretPresence, pub anthropic: SecretPresence, + pub typesafe: SecretPresence, } impl UpstreamAuthInfo { @@ -118,6 +119,10 @@ impl UpstreamAuthInfo { gateway.anthropic_auth_header.as_deref(), "ANTHROPIC_API_KEY", ), + typesafe: SecretPresence::from_effective_provider_auth( + gateway.typesafe_auth_header.as_deref(), + "TYPESAFE_API_KEY", + ), } } @@ -125,6 +130,7 @@ impl UpstreamAuthInfo { Self { openai: SecretPresence::Unknown, anthropic: SecretPresence::Unknown, + typesafe: SecretPresence::Unknown, } } } diff --git a/crates/cli/src/diagnostics/render.rs b/crates/cli/src/diagnostics/render.rs index d930ed6ca..706c69641 100644 --- a/crates/cli/src/diagnostics/render.rs +++ b/crates/cli/src/diagnostics/render.rs @@ -156,9 +156,10 @@ pub(super) fn format_human_configuration(out: &mut String, report: &DoctorReport out.push_str(&format!(" {label:<11}{}\n", format_layer(layer))); } out.push_str(&format!( - " Upstream openai={} anthropic={}\n", + " Upstream openai={} anthropic={} typesafe={}\n", report.configuration.upstream_auth.openai.as_str(), - report.configuration.upstream_auth.anthropic.as_str() + report.configuration.upstream_auth.anthropic.as_str(), + report.configuration.upstream_auth.typesafe.as_str() )); if !matches!(report.configuration.resolution.status, Status::Pass) { out.push_str(&format!( diff --git a/crates/cli/src/gateway/mod.rs b/crates/cli/src/gateway/mod.rs index 4fa2a3aa4..ba828c01b 100644 --- a/crates/cli/src/gateway/mod.rs +++ b/crates/cli/src/gateway/mod.rs @@ -1081,15 +1081,113 @@ async fn forward_upstream_request( forwarding.source_route, ); let configured_auth_header = forwarding.configured_auth_header(effective.target_route); + if let Some(operational) = operational { + operational::upstream_started(operational, streaming); + } + let is_typesafe = matches!( + effective.target_route, + ProviderRoute::TypeSafeSystemOne | ProviderRoute::TypeSafeModels + ); + + let mut retry_count = 0_u32; + loop { + let upstream = build_upstream_attempt( + http, + method, + &effective, + &forwarding, + configured_auth_header, + retry_count, + ); + let response = send_upstream_attempt(upstream, operational).await; + + let response = match response { + Ok(response) => response, + Err(error) => { + if is_typesafe + && (error.is_timeout() || error.is_connect()) + && forwarding.typesafe_retry.can_retry_transport(retry_count) + { + let delay = forwarding.typesafe_retry.delay(None, retry_count); + retry_count += 1; + if let Some(operational) = operational { + operational::upstream_retry_scheduled( + operational, + "typesafe", + "transport", + retry_count, + delay.as_millis().try_into().unwrap_or(u64::MAX), + ); + } + tokio::time::sleep(delay).await; + continue; + } + if let Some(operational) = operational { + operational::upstream_failed(operational, "transport"); + } + return Err(error); + } + }; + if is_typesafe + && forwarding + .typesafe_retry + .should_retry_status(response.status(), retry_count) + { + let status = response.status(); + let delay = forwarding + .typesafe_retry + .delay(Some(response.headers()), retry_count); + retry_count += 1; + if let Some(operational) = operational { + operational::upstream_retry_scheduled( + operational, + "typesafe", + if status == StatusCode::TOO_MANY_REQUESTS { + "rate_limit" + } else { + "status" + }, + retry_count, + delay.as_millis().try_into().unwrap_or(u64::MAX), + ); + } + drop(response); + tokio::time::sleep(delay).await; + continue; + } + if let Some(operational) = operational { + operational::upstream_headers_received(operational, streaming); + } + return Ok(response); + } +} + +fn build_upstream_attempt( + http: &reqwest::Client, + method: &Method, + effective: &EffectiveUpstreamRequest, + forwarding: &ProviderForwarding, + configured_auth_header: Option<&str>, + retry_count: u32, +) -> reqwest::RequestBuilder { let mut upstream = http .request(method.clone(), &effective.url) .body(effective.body_bytes.clone()); for (name, value) in &effective.headers { - if should_forward_request_header(name, &effective.headers) { + if name.as_str() != "x-typesafe-retry-count" + && should_forward_request_header(name, &effective.headers) + { upstream = upstream.header(name, value); } } - upstream = inject_provider_auth( + let is_typesafe = matches!( + effective.target_route, + ProviderRoute::TypeSafeSystemOne | ProviderRoute::TypeSafeModels + ); + if is_typesafe && retry_count > 0 { + upstream = upstream.header("x-typesafe-retry-count", retry_count.to_string()); + } + inject_provider_auth( upstream, effective.target_route, &effective.headers, @@ -1098,30 +1196,28 @@ async fn forward_upstream_request( TargetCredentialPolicy::SourceOrEnvironment ) && forwarding.authorization.allow_environment_provider_auth, configured_auth_header, - ); - if let Some(operational) = operational { - operational::upstream_started(operational, streaming); - let request = upstream.send(); - tokio::pin!(request); - let response = tokio::select! { - response = &mut request => response, - _ = tokio::time::sleep(std::time::Duration::from_millis(UPSTREAM_RESPONSE_THRESHOLD_MILLIS)) => { - operational::upstream_delayed( - operational, - "upstream_headers_delayed", - UPSTREAM_RESPONSE_THRESHOLD_MILLIS, - ); - request.await - } - }; - if response.is_ok() { - operational::upstream_headers_received(operational, streaming); - } else { - operational::upstream_failed(operational, "transport"); + ) +} + +async fn send_upstream_attempt( + upstream: reqwest::RequestBuilder, + operational: Option<&OperationalContext>, +) -> Result { + let Some(operational) = operational else { + return upstream.send().await; + }; + let request = upstream.send(); + tokio::pin!(request); + tokio::select! { + response = &mut request => response, + _ = tokio::time::sleep(std::time::Duration::from_millis(UPSTREAM_RESPONSE_THRESHOLD_MILLIS)) => { + operational::upstream_delayed( + operational, + "upstream_headers_delayed", + UPSTREAM_RESPONSE_THRESHOLD_MILLIS, + ); + request.await } - response - } else { - upstream.send().await } } @@ -1384,6 +1480,9 @@ where ProviderRoute::AnthropicMessages | ProviderRoute::AnthropicCountTokens => { ("ANTHROPIC_API_KEY", "x-api-key") } + ProviderRoute::TypeSafeSystemOne | ProviderRoute::TypeSafeModels => { + ("TYPESAFE_API_KEY", http::header::AUTHORIZATION.as_str()) + } }; let Some(value) = env_lookup(env_var) else { return builder; @@ -1400,6 +1499,9 @@ where | ProviderRoute::OpenAiImagesGenerations | ProviderRoute::OpenAiModels => format!("Bearer {value}"), ProviderRoute::AnthropicMessages | ProviderRoute::AnthropicCountTokens => value, + ProviderRoute::TypeSafeSystemOne | ProviderRoute::TypeSafeModels => { + format!("Bearer {value}") + } }; builder.header(header_name, header_value) } @@ -1566,7 +1668,7 @@ fn http_failure(status: StatusCode, headers: &HeaderMap, body: &[u8]) -> Upstrea || normalized.contains("model_overloaded") { UpstreamFailureClass::ModelUnavailable - } else if matches!(status.as_u16(), 408 | 429 | 500 | 502 | 503 | 504) { + } else if matches!(status.as_u16(), 408 | 429 | 500 | 502 | 503 | 504 | 529) { UpstreamFailureClass::RetryableStatus } else if status.is_client_error() { UpstreamFailureClass::InvalidRequest @@ -1627,7 +1729,7 @@ fn bounded_error_body(body: &[u8]) -> String { String::from_utf8_lossy(&body[..body.len().min(MAX_UPSTREAM_ERROR_BODY_BYTES)]).into_owned() } -/// Proxies OpenAI model-list requests without creating LLM runtime events. +/// Proxies provider model-list requests without creating LLM runtime events. /// /// The route is registered as GET-only but still verifies the method so direct tests or future /// router changes return a 405 instead of forwarding a nonsensical request upstream. @@ -1644,7 +1746,14 @@ pub(crate) async fn models( Body::empty(), ); } - let provider = ProviderRoute::OpenAiModels; + let provider = ProviderRoute::from_path(parts.uri.path()) + .filter(|provider| { + matches!( + provider, + ProviderRoute::OpenAiModels | ProviderRoute::TypeSafeModels + ) + }) + .ok_or_else(|| CliError::InvalidPayload("unsupported model-list route".into()))?; let configured_auth_header = provider.configured_auth_header(&state.config); let path_and_query = parts .uri @@ -1694,24 +1803,61 @@ pub(crate) async fn models( allow_environment_provider_auth, configured_auth_header, ); - let mut upstream = if named_by_client { - state.http_no_redirect.get(upstream_url) - } else { - state.http.get(upstream_url) - }; - for (name, value) in &sanitized { - if should_forward_request_header(name, &sanitized) { - upstream = upstream.header(name, value); + let mut retry_count = 0_u32; + let upstream_response = loop { + let mut upstream = if named_by_client { + state.http_no_redirect.get(&upstream_url) + } else { + state.http.get(&upstream_url) + }; + for (name, value) in &sanitized { + if name.as_str() != "x-typesafe-retry-count" + && should_forward_request_header(name, &sanitized) + { + upstream = upstream.header(name, value); + } } - } - upstream = inject_provider_auth( - upstream, - provider, - &sanitized, - allow_environment_provider_auth, - configured_auth_header, - ); - let upstream_response = upstream.send().await?; + if provider == ProviderRoute::TypeSafeModels && retry_count > 0 { + upstream = upstream.header("x-typesafe-retry-count", retry_count.to_string()); + } + upstream = inject_provider_auth( + upstream, + provider, + &sanitized, + allow_environment_provider_auth, + configured_auth_header, + ); + let response = match upstream.send().await { + Ok(response) => response, + Err(error) + if provider == ProviderRoute::TypeSafeModels + && (error.is_timeout() || error.is_connect()) + && state.config.typesafe_retry.can_retry_transport(retry_count) => + { + let delay = state.config.typesafe_retry.delay(None, retry_count); + retry_count += 1; + tokio::time::sleep(delay).await; + continue; + } + Err(error) => return Err(error.into()), + }; + if provider == ProviderRoute::TypeSafeModels + && state + .config + .typesafe_retry + .should_retry_status(response.status(), retry_count) + { + let delay = state + .config + .typesafe_retry + .delay(Some(response.headers()), retry_count); + retry_count += 1; + drop(response); + tokio::time::sleep(delay).await; + continue; + } + break response; + }; let status = upstream_response.status(); let headers = response_headers(upstream_response.headers()); let bytes = upstream_response.bytes().await?; diff --git a/crates/cli/src/gateway/request.rs b/crates/cli/src/gateway/request.rs index fbdb38cfc..7f50d251a 100644 --- a/crates/cli/src/gateway/request.rs +++ b/crates/cli/src/gateway/request.rs @@ -60,6 +60,15 @@ pub(super) async fn prepare_gateway_request( ) .and_then(|body| serde_json::from_slice::(&body).ok()) .unwrap_or(Value::Null); + if provider == ProviderRoute::TypeSafeSystemOne + && request_json + .as_object() + .is_some_and(|object| object.contains_key("stream")) + { + return Err(CliError::InvalidPayload( + "TypeSafe System One is non-streaming; remove the stream field".into(), + )); + } let path_and_query = parts .uri .path_and_query() diff --git a/crates/cli/src/gateway/routes.rs b/crates/cli/src/gateway/routes.rs index c3f18e741..3ce724b90 100644 --- a/crates/cli/src/gateway/routes.rs +++ b/crates/cli/src/gateway/routes.rs @@ -13,6 +13,8 @@ pub(super) enum ProviderRoute { OpenAiModels, AnthropicMessages, AnthropicCountTokens, + TypeSafeSystemOne, + TypeSafeModels, } #[derive(Clone)] @@ -21,6 +23,8 @@ pub(super) struct ProviderForwarding { pub(super) authorization: crate::provider_auth::ProviderRequestAuthorization, openai_auth_header: Option, anthropic_auth_header: Option, + typesafe_auth_header: Option, + pub(super) typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy, } impl ProviderForwarding { @@ -34,6 +38,8 @@ impl ProviderForwarding { authorization, openai_auth_header: config.openai_auth_header.clone(), anthropic_auth_header: config.anthropic_auth_header.clone(), + typesafe_auth_header: config.typesafe_auth_header.clone(), + typesafe_retry: config.typesafe_retry, } } @@ -42,6 +48,7 @@ impl ProviderForwarding { route, self.openai_auth_header.as_deref(), self.anthropic_auth_header.as_deref(), + self.typesafe_auth_header.as_deref(), ) } } @@ -61,6 +68,10 @@ impl ProviderRoute { "/v1/models" => Some(Self::OpenAiModels), "/v1/messages" => Some(Self::AnthropicMessages), "/v1/messages/count_tokens" => Some(Self::AnthropicCountTokens), + "/systemone" | "/v1/systemone" | "/typesafe/systemone" | "/typesafe/v1/systemone" => { + Some(Self::TypeSafeSystemOne) + } + "/typesafe/models" | "/typesafe/v1/models" => Some(Self::TypeSafeModels), _ => None, } } @@ -86,6 +97,12 @@ impl ProviderRoute { "anthropic_count_tokens" | "anthropic.count_tokens" | "/v1/messages/count_tokens" => { Some(Self::AnthropicCountTokens) } + "typesafe_system_one" | "typesafe.system_one" | "/v1/systemone" => { + Some(Self::TypeSafeSystemOne) + } + "typesafe_models" | "typesafe.models" | "/typesafe/models" | "/typesafe/v1/models" => { + Some(Self::TypeSafeModels) + } _ => None, } } @@ -95,7 +112,11 @@ impl ProviderRoute { Self::OpenAiResponses => Some(ProviderSurface::OpenAIResponses), Self::OpenAiChatCompletions => Some(ProviderSurface::OpenAIChat), Self::AnthropicMessages => Some(ProviderSurface::AnthropicMessages), - Self::AnthropicCountTokens | Self::OpenAiImagesGenerations | Self::OpenAiModels => None, + Self::TypeSafeSystemOne => Some(ProviderSurface::TypeSafeSystemOne), + Self::AnthropicCountTokens + | Self::OpenAiImagesGenerations + | Self::OpenAiModels + | Self::TypeSafeModels => None, } } @@ -122,6 +143,7 @@ impl ProviderRoute { Self::AnthropicMessages | Self::AnthropicCountTokens => { config.anthropic_base_url.as_str() } + Self::TypeSafeSystemOne | Self::TypeSafeModels => config.typesafe_base_url.as_str(), }; self.upstream_url_with_base(base, path_and_query) } @@ -134,6 +156,7 @@ impl ProviderRoute { self, config.openai_auth_header.as_deref(), config.anthropic_auth_header.as_deref(), + config.typesafe_auth_header.as_deref(), ) } @@ -147,6 +170,11 @@ impl ProviderRoute { } fn canonical_path_and_query(self, path_and_query: &str) -> String { + if matches!(self, Self::TypeSafeSystemOne | Self::TypeSafeModels) + && let Some(suffix) = path_and_query.strip_prefix("/typesafe") + { + return suffix.to_string(); + } if self == Self::OpenAiResponses && let Some(suffix) = path_and_query.strip_prefix("/backend-api/codex/responses") { @@ -166,6 +194,8 @@ impl ProviderRoute { Self::OpenAiModels => GatewayRouteKind::OpenAiModels, Self::AnthropicMessages => GatewayRouteKind::AnthropicMessages, Self::AnthropicCountTokens => GatewayRouteKind::AnthropicCountTokens, + Self::TypeSafeSystemOne => GatewayRouteKind::TypeSafeSystemOne, + Self::TypeSafeModels => GatewayRouteKind::TypeSafeModels, } } } @@ -174,6 +204,7 @@ fn configured_auth_header<'a>( route: ProviderRoute, openai_auth_header: Option<&'a str>, anthropic_auth_header: Option<&'a str>, + typesafe_auth_header: Option<&'a str>, ) -> Option<&'a str> { match route { ProviderRoute::OpenAiResponses @@ -183,6 +214,7 @@ fn configured_auth_header<'a>( ProviderRoute::AnthropicMessages | ProviderRoute::AnthropicCountTokens => { anthropic_auth_header } + ProviderRoute::TypeSafeSystemOne | ProviderRoute::TypeSafeModels => typesafe_auth_header, } } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 7c3b073d4..ccc1dd046 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -28,6 +28,7 @@ mod process; mod provider_auth; mod server; mod sessions; +mod typesafe_retry; #[cfg(test)] #[path = "../tests/coverage/shared/hook_assertions.rs"] diff --git a/crates/cli/src/operational.rs b/crates/cli/src/operational.rs index 7d7d54950..f9bb12445 100644 --- a/crates/cli/src/operational.rs +++ b/crates/cli/src/operational.rs @@ -377,6 +377,26 @@ pub(crate) fn upstream_status(context: &OperationalContext, status_code: u16) { ); } +pub(crate) fn upstream_retry_scheduled( + context: &OperationalContext, + provider: &'static str, + reason: &'static str, + retry_number: u32, + delay_millis: u64, +) { + operational_log!( + log::Level::Warn, + "upstream_retry_scheduled", + context; + boundary = "upstream", + provider, + reason, + retry_number, + delay_millis, + elapsed_millis = context.elapsed_millis() + ); +} + pub(crate) fn upstream_failed(context: &OperationalContext, error_kind: &'static str) { operational_log!( log::Level::Error, diff --git a/crates/cli/src/process/launcher.rs b/crates/cli/src/process/launcher.rs index 9458e398e..e220090c5 100644 --- a/crates/cli/src/process/launcher.rs +++ b/crates/cli/src/process/launcher.rs @@ -614,6 +614,8 @@ impl PreparedAgentLaunch { resolved.gateway.anthropic_base_url ); println!("anthropic_auth = {}", upstream_auth.anthropic.as_str()); + println!("typesafe_base_url = {}", resolved.gateway.typesafe_base_url); + println!("typesafe_auth = {}", upstream_auth.typesafe.as_str()); println!( "max_hook_payload_bytes = {}", resolved.gateway.max_hook_payload_bytes diff --git a/crates/cli/src/process/types.rs b/crates/cli/src/process/types.rs index d86aea678..7fee43231 100644 --- a/crates/cli/src/process/types.rs +++ b/crates/cli/src/process/types.rs @@ -11,6 +11,7 @@ pub(crate) struct RunOverrides { pub(crate) config: Option, pub(crate) openai_base_url: Option, pub(crate) anthropic_base_url: Option, + pub(crate) typesafe_base_url: Option, pub(crate) session_metadata: Option, pub(crate) plugin_config_path: Option, pub(crate) dry_run: bool, diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index 366390bfc..9e004523c 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -673,6 +673,12 @@ fn router_with_state(state: AppState) -> Router { .route("/v1/images/generations", post(gateway::images_generations)) .route("/v1/messages", post(gateway::passthrough)) .route("/v1/messages/count_tokens", post(gateway::passthrough)) + .route("/systemone", post(gateway::passthrough)) + .route("/v1/systemone", post(gateway::passthrough)) + .route("/typesafe/systemone", post(gateway::passthrough)) + .route("/typesafe/v1/systemone", post(gateway::passthrough)) + .route("/typesafe/models", get(gateway::models)) + .route("/typesafe/v1/models", get(gateway::models)) .route("/v1/models", get(gateway::models)) .layer(middleware::from_fn(responses_websocket_fallback)) .layer(DefaultBodyLimit::max(max_hook_payload_bytes)) diff --git a/crates/cli/src/server/types.rs b/crates/cli/src/server/types.rs index 25f63d3c3..230043ecb 100644 --- a/crates/cli/src/server/types.rs +++ b/crates/cli/src/server/types.rs @@ -10,6 +10,7 @@ pub(crate) struct GatewayOverrides { pub(crate) bind: Option, pub(crate) openai_base_url: Option, pub(crate) anthropic_base_url: Option, + pub(crate) typesafe_base_url: Option, pub(crate) plugin_config_path: Option, pub(crate) ready_file: Option, pub(crate) max_hook_payload_bytes: Option, diff --git a/crates/cli/src/typesafe_retry.rs b/crates/cli/src/typesafe_retry.rs new file mode 100644 index 000000000..621067898 --- /dev/null +++ b/crates/cli/src/typesafe_retry.rs @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Retry policy shared by direct gateways and daemon-managed workers for TypeSafe APIs. + +use std::time::{Duration, SystemTime}; + +use http::HeaderMap; +use ring::rand::{SecureRandom, SystemRandom}; + +pub(crate) const DEFAULT_MAX_RETRIES: u32 = 2; +pub(crate) const DEFAULT_MAX_SERVER_DELAY_MILLIS: u64 = 60_000; +const INITIAL_BACKOFF_MILLIS: u64 = 500; +const MAX_BACKOFF_MILLIS: u64 = 5_000; +const JITTER_RATIO: f64 = 0.25; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct TypeSafeRetryPolicy { + pub(crate) max_retries: u32, + pub(crate) max_server_delay: Duration, +} + +impl Default for TypeSafeRetryPolicy { + fn default() -> Self { + Self { + max_retries: DEFAULT_MAX_RETRIES, + max_server_delay: Duration::from_millis(DEFAULT_MAX_SERVER_DELAY_MILLIS), + } + } +} + +impl TypeSafeRetryPolicy { + pub(crate) fn should_retry_status(self, status: http::StatusCode, retry_count: u32) -> bool { + retry_count < self.max_retries + && (status == http::StatusCode::REQUEST_TIMEOUT + || status == http::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error()) + } + + pub(crate) fn can_retry_transport(self, retry_count: u32) -> bool { + retry_count < self.max_retries + } + + pub(crate) fn delay(self, headers: Option<&HeaderMap>, retry_count: u32) -> Duration { + if let Some(delay) = headers.and_then(server_retry_delay) + && delay <= self.max_server_delay + { + return delay; + } + jittered_backoff(retry_count, random_unit_interval()) + } +} + +fn server_retry_delay(headers: &HeaderMap) -> Option { + if let Some(delay) = headers + .get("retry-after-ms") + .and_then(|value| value.to_str().ok()) + .and_then(parse_millis) + { + return Some(delay); + } + let value = headers + .get(http::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim(); + if let Ok(seconds) = value.parse::() { + return finite_nonnegative_duration(seconds * 1_000.0); + } + let retry_at = httpdate::parse_http_date(value).ok()?; + Some( + retry_at + .duration_since(SystemTime::now()) + .unwrap_or_default(), + ) +} + +fn parse_millis(value: &str) -> Option { + finite_nonnegative_duration(value.trim().parse::().ok()?) +} + +fn finite_nonnegative_duration(millis: f64) -> Option { + (millis.is_finite() && millis >= 0.0).then(|| Duration::from_secs_f64(millis / 1_000.0)) +} + +fn jittered_backoff(retry_count: u32, unit: f64) -> Duration { + let base = INITIAL_BACKOFF_MILLIS + .saturating_mul(1_u64 << retry_count.min(3)) + .min(MAX_BACKOFF_MILLIS) as f64; + // Match both official SDKs: jitter is randomly subtracted from the exponential delay, + // producing 75%-100% of the unjittered value at the default ratio. + let factor = 1.0 - (JITTER_RATIO * unit.clamp(0.0, 1.0)); + Duration::from_millis((base * factor).round() as u64) +} + +fn random_unit_interval() -> f64 { + let mut bytes = [0_u8; 8]; + if SystemRandom::new().fill(&mut bytes).is_err() { + return 0.5; + } + u64::from_le_bytes(bytes) as f64 / u64::MAX as f64 +} diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index 4e53b971a..512347b6b 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -79,6 +79,7 @@ fn infers_agent_from_command_or_uses_override() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -135,6 +136,7 @@ fn uses_configured_command_when_no_argv_is_supplied() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -155,6 +157,7 @@ fn inference_failure_has_actionable_message() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -180,6 +183,7 @@ fn missing_command_without_agent_errors() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -203,6 +207,7 @@ fn agent_without_configured_command_falls_back_to_default_binary() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -224,6 +229,7 @@ fn agent_with_passthrough_args_appends_to_configured_command() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -961,6 +967,7 @@ fn invocation_resolves_wrapper_host_before_appending_pass_through_arguments() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -1713,6 +1720,7 @@ async fn run_starts_gateway_injects_env_and_returns_agent_exit_code() { config: Some(config), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -1754,6 +1762,7 @@ async fn dry_run_does_not_spawn_agent() { config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: true, @@ -1785,6 +1794,7 @@ async fn transparent_launcher_does_not_initialize_logging_sinks_directly() { config: Some(config_path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: true, @@ -1851,6 +1861,7 @@ entrypoint = "acme.worker:create_plugin" config: Some(config_path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: true, diff --git a/crates/cli/tests/coverage/commands/configure_editor_tests.rs b/crates/cli/tests/coverage/commands/configure_editor_tests.rs index 5b5dfd30f..84c759ac9 100644 --- a/crates/cli/tests/coverage/commands/configure_editor_tests.rs +++ b/crates/cli/tests/coverage/commands/configure_editor_tests.rs @@ -13,7 +13,7 @@ fn document(contents: &str) -> ConfigDocument { #[test] fn document_preserves_toml_and_redacts_standard_inline_and_dotted_auth_headers() { let mut standard = document( - "# keep this comment\n[agents.codex]\ncommand = \"codex\"\n\n[upstream]\nopenai_auth_header = \"Bearer secret\"\nanthropic_auth_header = \"Basic secret\"\n", + "# keep this comment\n[agents.codex]\ncommand = \"codex\"\n\n[upstream]\nopenai_auth_header = \"Bearer secret\"\nanthropic_auth_header = \"Basic secret\"\ntypesafe_auth_header = \"Bearer jev-secret\"\n", ); standard .set_positive_integer("gateway", "max_hook_payload_bytes", 42) @@ -25,10 +25,11 @@ fn document_preserves_toml_and_redacts_standard_inline_and_dotted_auth_headers() assert!(preview.contains("")); assert!(!preview.contains("Bearer secret")); assert!(!preview.contains("Basic secret")); + assert!(!preview.contains("jev-secret")); assert!(standard.document.to_string().contains("Bearer secret")); let mut inline = document( - "upstream = { openai_auth_header = \"Bearer inline\", anthropic_auth_header = \"Basic inline\" }\n", + "upstream = { openai_auth_header = \"Bearer inline\", anthropic_auth_header = \"Basic inline\", typesafe_auth_header = \"Bearer jev-inline\" }\n", ); assert_eq!(inline.secret_summary("openai_auth_header"), "configured"); inline @@ -42,6 +43,7 @@ fn document_preserves_toml_and_redacts_standard_inline_and_dotted_auth_headers() assert!(!preview.contains("Bearer inline")); assert!(!preview.contains("Bearer replacement")); assert!(!preview.contains("Basic inline")); + assert!(!preview.contains("jev-inline")); let dotted = document("upstream.openai_auth_header = \"Bearer dotted\"\n"); assert_eq!(dotted.secret_summary("openai_auth_header"), "configured"); @@ -257,6 +259,7 @@ fn global_document_rejects_authorization_headers() { "[upstream]\nopenai_auth_header = \"Bearer secret\"\n", "upstream = { anthropic_auth_header = \"Bearer secret\" }\n", "upstream.openai_auth_header = \"Bearer secret\"\n", + "[upstream]\ntypesafe_auth_header = \"Bearer jev-secret\"\n", ] .into_iter() .enumerate() diff --git a/crates/cli/tests/coverage/daemon/routes_tests.rs b/crates/cli/tests/coverage/daemon/routes_tests.rs index 9d6044d6b..ec3579559 100644 --- a/crates/cli/tests/coverage/daemon/routes_tests.rs +++ b/crates/cli/tests/coverage/daemon/routes_tests.rs @@ -14,6 +14,23 @@ fn classifies_only_supported_public_paths() { Some(PublicRoute::Hook(HookRoute::Codex)) ); assert_eq!(PublicRoute::from_path("/admin"), None); + assert_eq!( + PublicRoute::from_path("/typesafe/v1/models"), + Some(PublicRoute::Provider(ProviderRoute::TypeSafe)) + ); +} + +#[test] +fn composes_namespaced_typesafe_paths_for_the_sdk() { + let config = GatewayConfig::default(); + assert_eq!( + ProviderRoute::TypeSafe.upstream_url(&config, "/typesafe/v1/systemone?x=1"), + "https://api.typesafe.ai/v1/systemone?x=1" + ); + assert_eq!( + ProviderRoute::TypeSafe.upstream_url(&config, "/typesafe/v1/models"), + "https://api.typesafe.ai/v1/models" + ); } #[test] diff --git a/crates/cli/tests/coverage/daemon/server_tests.rs b/crates/cli/tests/coverage/daemon/server_tests.rs index 630ecf441..fbc199408 100644 --- a/crates/cli/tests/coverage/daemon/server_tests.rs +++ b/crates/cli/tests/coverage/daemon/server_tests.rs @@ -1018,6 +1018,10 @@ fn pass_through_auth_injection_supports_environment_and_anthropic_configuration( "ANTHROPIC_API_KEY", Some(std::ffi::OsStr::new(" anthropic-env ")), ), + ( + "TYPESAFE_API_KEY", + Some(std::ffi::OsStr::new(" typesafe-env ")), + ), ]); let mut openai = HeaderMap::new(); inject_provider_auth( @@ -1035,12 +1039,22 @@ fn pass_through_auth_injection_supports_environment_and_anthropic_configuration( ); assert_eq!(anthropic["x-api-key"], "anthropic-env"); + let mut typesafe = HeaderMap::new(); + inject_provider_auth( + &mut typesafe, + ProviderRoute::TypeSafe, + &GatewayConfig::default(), + ); + assert_eq!(typesafe[AUTHORIZATION], "Bearer typesafe-env"); + let mut configured = HeaderMap::new(); inject_provider_auth( &mut configured, ProviderRoute::Anthropic, &GatewayConfig { anthropic_auth_header: Some("configured".into()), + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, ..GatewayConfig::default() }, ); diff --git a/crates/cli/tests/coverage/daemon/worker_managed_tests.rs b/crates/cli/tests/coverage/daemon/worker_managed_tests.rs index de8d4a21b..674275a80 100644 --- a/crates/cli/tests/coverage/daemon/worker_managed_tests.rs +++ b/crates/cli/tests/coverage/daemon/worker_managed_tests.rs @@ -890,6 +890,139 @@ async fn prepared_requests_and_both_dispatch_paths_preserve_provider_contracts() server.abort(); } +#[tokio::test] +async fn daemon_observed_typesafe_dispatch_retries_with_sdk_semantics() { + let attempts = Arc::new(AtomicUsize::new(0)); + let retry_headers = Arc::new(std::sync::Mutex::new(Vec::>::new())); + let app = Router::new().route( + "/v1/systemone", + post({ + let attempts = Arc::clone(&attempts); + let retry_headers = Arc::clone(&retry_headers); + move |headers: HeaderMap| { + let attempts = Arc::clone(&attempts); + let retry_headers = Arc::clone(&retry_headers); + async move { + retry_headers.lock().unwrap().push( + headers + .get("x-typesafe-retry-count") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned), + ); + let attempt = attempts.fetch_add(1, AtomicOrdering::SeqCst); + if attempt < 2 { + Response::builder() + .status(if attempt == 0 { 429 } else { 529 }) + .header(http::header::RETRY_AFTER, "0") + .body(Body::empty()) + .unwrap() + } else { + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + r#"{"model":"jev-1.13.0","answers":{"q":{"type":"noul","noul":0.9}},"usage":{"input_tokens":3,"output_tokens":0}}"#, + )) + .unwrap() + } + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let config = GatewayConfig { + typesafe_base_url: format!("http://{address}/v1"), + ..GatewayConfig::default() + }; + let request = Request::post("/typesafe/v1/systemone") + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + r#"{"model":"jev-latest","state":"candidate","questions":{"q":{"type":"noul"}}}"#, + )) + .unwrap(); + let prepared = PreparedProviderRequest::read(request, &config) + .await + .unwrap(); + let (response, observation) = dispatch_observed( + pooled_client().unwrap(), + prepared, + ProviderRoute::TypeSafe, + None, + &config, + DEFAULT_OBSERVATION_CAPTURE_BYTES, + OperationalContext::new(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["model"], + "jev-1.13.0" + ); + let observed = observation + .finish(ProviderSurface::TypeSafeSystemOne, false) + .await; + assert_eq!(observed.value.unwrap()["answers"]["q"]["noul"], 0.9); + assert_eq!(attempts.load(AtomicOrdering::SeqCst), 3); + assert_eq!( + *retry_headers.lock().unwrap(), + vec![None, Some("1".into()), Some("2".into())] + ); + server.abort(); +} + +#[tokio::test] +async fn daemon_unmanaged_typesafe_model_catalog_retries_and_strips_namespace() { + let attempts = Arc::new(AtomicUsize::new(0)); + let app = Router::new().route( + "/v1/models", + axum::routing::get({ + let attempts = Arc::clone(&attempts); + move || { + let attempts = Arc::clone(&attempts); + async move { + if attempts.fetch_add(1, AtomicOrdering::SeqCst) == 0 { + Response::builder() + .status(StatusCode::TOO_MANY_REQUESTS) + .header(http::header::RETRY_AFTER, "0") + .body(Body::empty()) + .unwrap() + } else { + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(r#"{"models":[{"name":"jev-latest"}]}"#)) + .unwrap() + } + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let config = GatewayConfig { + typesafe_base_url: format!("http://{address}/v1"), + ..GatewayConfig::default() + }; + let response = dispatch_unmanaged( + pooled_client().unwrap(), + Request::get("/typesafe/v1/models") + .body(Body::empty()) + .unwrap(), + ProviderRoute::TypeSafe, + &config, + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(attempts.load(AtomicOrdering::SeqCst), 2); + server.abort(); +} + #[tokio::test] async fn prepared_request_enforces_body_limit_and_normalizes_non_json_payloads() { let limited = GatewayConfig { diff --git a/crates/cli/tests/coverage/daemon/worker_runtime_tests.rs b/crates/cli/tests/coverage/daemon/worker_runtime_tests.rs index b1a3bc0dc..0fa9a389f 100644 --- a/crates/cli/tests/coverage/daemon/worker_runtime_tests.rs +++ b/crates/cli/tests/coverage/daemon/worker_runtime_tests.rs @@ -639,6 +639,10 @@ fn provider_auth_supports_both_configured_and_environment_credentials() { "ANTHROPIC_API_KEY", Some(std::ffi::OsStr::new(" anthropic-env ")), ), + ( + "TYPESAFE_API_KEY", + Some(std::ffi::OsStr::new(" typesafe-env ")), + ), ]); let mut openai = HeaderMap::new(); inject_provider_auth( @@ -656,12 +660,22 @@ fn provider_auth_supports_both_configured_and_environment_credentials() { ); assert_eq!(anthropic["x-api-key"], "anthropic-env"); + let mut typesafe = HeaderMap::new(); + inject_provider_auth( + &mut typesafe, + ProviderRoute::TypeSafe, + &GatewayConfig::default(), + ); + assert_eq!(typesafe[AUTHORIZATION], "Bearer typesafe-env"); + let mut configured = HeaderMap::new(); inject_provider_auth( &mut configured, ProviderRoute::Anthropic, &GatewayConfig { anthropic_auth_header: Some("configured-anthropic".into()), + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, ..GatewayConfig::default() }, ); diff --git a/crates/cli/tests/coverage/shared/config_tests.rs b/crates/cli/tests/coverage/shared/config_tests.rs index 437905018..c4316240b 100644 --- a/crates/cli/tests/coverage/shared/config_tests.rs +++ b/crates/cli/tests/coverage/shared/config_tests.rs @@ -227,6 +227,8 @@ struct PluginConfigDiscoveryScope { previous_openai_auth_header: Option, previous_anthropic_base_url: Option, previous_anthropic_auth_header: Option, + previous_typesafe_base_url: Option, + previous_typesafe_auth_header: Option, previous_bootstrap_fingerprint: Option, previous_plugin_idle_timeout: Option, previous_plugin_heartbeat_interval: Option, @@ -246,6 +248,8 @@ impl PluginConfigDiscoveryScope { let previous_openai_auth_header = std::env::var_os("NEMO_RELAY_OPENAI_AUTH_HEADER"); let previous_anthropic_base_url = std::env::var_os("NEMO_RELAY_ANTHROPIC_BASE_URL"); let previous_anthropic_auth_header = std::env::var_os("NEMO_RELAY_ANTHROPIC_AUTH_HEADER"); + let previous_typesafe_base_url = std::env::var_os("NEMO_RELAY_TYPESAFE_BASE_URL"); + let previous_typesafe_auth_header = std::env::var_os("NEMO_RELAY_TYPESAFE_AUTH_HEADER"); let previous_bootstrap_fingerprint = std::env::var_os(BOOTSTRAP_FINGERPRINT_ENV); let previous_plugin_idle_timeout = std::env::var_os(PLUGIN_IDLE_TIMEOUT_ENV); let previous_plugin_heartbeat_interval = std::env::var_os(PLUGIN_HEARTBEAT_INTERVAL_ENV); @@ -258,6 +262,8 @@ impl PluginConfigDiscoveryScope { std::env::remove_var("NEMO_RELAY_OPENAI_AUTH_HEADER"); std::env::remove_var("NEMO_RELAY_ANTHROPIC_BASE_URL"); std::env::remove_var("NEMO_RELAY_ANTHROPIC_AUTH_HEADER"); + std::env::remove_var("NEMO_RELAY_TYPESAFE_BASE_URL"); + std::env::remove_var("NEMO_RELAY_TYPESAFE_AUTH_HEADER"); std::env::remove_var(BOOTSTRAP_FINGERPRINT_ENV); std::env::remove_var(PLUGIN_IDLE_TIMEOUT_ENV); std::env::remove_var(PLUGIN_HEARTBEAT_INTERVAL_ENV); @@ -274,6 +280,8 @@ impl PluginConfigDiscoveryScope { previous_openai_auth_header, previous_anthropic_base_url, previous_anthropic_auth_header, + previous_typesafe_base_url, + previous_typesafe_auth_header, previous_bootstrap_fingerprint, previous_plugin_idle_timeout, previous_plugin_heartbeat_interval, @@ -304,6 +312,14 @@ impl PluginConfigDiscoveryScope { } } + fn set_typesafe_config(&self, base_url: &str, auth_header: &str) { + // SAFETY: This scope holds the process-wide environment mutex. + unsafe { + std::env::set_var("NEMO_RELAY_TYPESAFE_BASE_URL", base_url); + std::env::set_var("NEMO_RELAY_TYPESAFE_AUTH_HEADER", auth_header); + } + } + #[cfg(feature = "__skip-implicit-config")] fn skip_implicit_config(&self) { // SAFETY: This scope holds the process-wide environment mutex. @@ -341,6 +357,14 @@ impl Drop for PluginConfigDiscoveryScope { Some(value) => std::env::set_var("NEMO_RELAY_ANTHROPIC_AUTH_HEADER", value), None => std::env::remove_var("NEMO_RELAY_ANTHROPIC_AUTH_HEADER"), } + match self.previous_typesafe_base_url.take() { + Some(value) => std::env::set_var("NEMO_RELAY_TYPESAFE_BASE_URL", value), + None => std::env::remove_var("NEMO_RELAY_TYPESAFE_BASE_URL"), + } + match self.previous_typesafe_auth_header.take() { + Some(value) => std::env::set_var("NEMO_RELAY_TYPESAFE_AUTH_HEADER", value), + None => std::env::remove_var("NEMO_RELAY_TYPESAFE_AUTH_HEADER"), + } match self.previous_bootstrap_fingerprint.take() { Some(value) => std::env::set_var(BOOTSTRAP_FINGERPRINT_ENV, value), None => std::env::remove_var(BOOTSTRAP_FINGERPRINT_ENV), @@ -594,6 +618,9 @@ fn config() -> GatewayConfig { openai_auth_header: None, anthropic_base_url: "http://anthropic".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -607,6 +634,7 @@ fn provider_auth_headers_default_to_unset() { assert!(config.openai_auth_header.is_none()); assert!(config.anthropic_auth_header.is_none()); + assert!(config.typesafe_auth_header.is_none()); } fn effective_plugin_toml_sources_without_system( @@ -931,6 +959,10 @@ openai_base_url = "http://openai" openai_auth_header = "Bearer openai-file" anthropic_base_url = "http://anthropic" anthropic_auth_header = "Basic anthropic-file" +typesafe_base_url = "http://typesafe/v1" +typesafe_auth_header = "Bearer typesafe-file" +typesafe_max_retries = 4 +typesafe_max_retry_delay_ms = 2500 [gateway] max_hook_payload_bytes = 12345 @@ -950,6 +982,7 @@ command = "codex --approval-mode never" config: Some(path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -970,6 +1003,16 @@ command = "codex --approval-mode never" resolved.gateway.anthropic_auth_header.as_deref(), Some("Basic anthropic-file") ); + assert_eq!(resolved.gateway.typesafe_base_url, "http://typesafe/v1"); + assert_eq!( + resolved.gateway.typesafe_auth_header.as_deref(), + Some("Bearer typesafe-file") + ); + assert_eq!(resolved.gateway.typesafe_retry.max_retries, 4); + assert_eq!( + resolved.gateway.typesafe_retry.max_server_delay, + std::time::Duration::from_millis(2500) + ); assert_eq!(resolved.gateway.max_hook_payload_bytes, 12345); assert_eq!(resolved.gateway.max_passthrough_body_bytes, 67890); assert_eq!(resolved.gateway.metadata, None); @@ -1029,6 +1072,43 @@ anthropic_auth_header = "Basic anthropic-file" ); } +#[test] +fn typesafe_environment_overrides_file_values() { + let temp = tempfile::tempdir().unwrap(); + let xdg = temp.path().join("xdg"); + std::fs::create_dir_all(&xdg).unwrap(); + let scope = PluginConfigDiscoveryScope::enter(temp.path(), &xdg); + let path = temp.path().join("config.toml"); + std::fs::write( + &path, + r#" +[upstream] +typesafe_base_url = "https://file.example/v1" +typesafe_auth_header = "Bearer file-secret" +"#, + ) + .unwrap(); + scope.set_typesafe_config( + "https://environment.example/v1", + " Bearer environment-secret ", + ); + + let resolved = resolve_server_config(&GatewayOverrides { + config: Some(path), + ..GatewayOverrides::default() + }) + .unwrap(); + + assert_eq!( + resolved.gateway.typesafe_base_url, + "https://environment.example/v1" + ); + assert_eq!( + resolved.gateway.typesafe_auth_header.as_deref(), + Some("Bearer environment-secret") + ); +} + #[test] fn ignored_project_endpoints_do_not_clear_user_provider_auth_headers() { let temp = tempfile::tempdir().unwrap(); @@ -1226,6 +1306,7 @@ fn explicit_config_must_exist() { config: Some(path.clone()), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: true, @@ -1308,6 +1389,7 @@ fn unreadable_config_errors_include_the_source_path() { config: Some(config_path.clone()), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: true, @@ -1366,6 +1448,7 @@ fn legacy_observability_config_sections_fail_clearly() { config: Some(path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -1420,6 +1503,7 @@ mode = "overwrite" config: Some(config_path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -2391,6 +2475,7 @@ fn plugin_config_path_overrides_sibling_plugin_file() { config: Some(config_path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: Some(override_path), dry_run: true, @@ -2428,6 +2513,7 @@ openai_auth_header = "Bearer file-openai" config: Some(path), openai_base_url: Some("http://cli-openai".into()), anthropic_base_url: None, + typesafe_base_url: None, session_metadata: Some(r#"{"team":"cli"}"#.into()), plugin_config_path: None, dry_run: false, @@ -2466,6 +2552,7 @@ openai_auth_header = "Bearer file-openai" config: None, openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, @@ -2502,6 +2589,7 @@ anthropic_auth_header = "Basic file-anthropic" bind: Some("127.0.0.1:0".parse().unwrap()), openai_base_url: Some("http://cli-openai".into()), anthropic_base_url: Some("http://cli-anthropic".into()), + typesafe_base_url: None, plugin_config_path: None, ready_file: None, max_hook_payload_bytes: Some(222), @@ -3748,6 +3836,7 @@ fn run_resolution_applies_all_run_overrides() { config: Some(config_path), openai_base_url: Some("http://run-openai".into()), anthropic_base_url: Some("http://run-anthropic".into()), + typesafe_base_url: None, session_metadata: Some(r#"{"team":"run"}"#.into()), plugin_config_path: None, dry_run: false, @@ -3790,6 +3879,7 @@ allowed = false config: Some(config_path), openai_base_url: None, anthropic_base_url: None, + typesafe_base_url: None, session_metadata: None, plugin_config_path: None, dry_run: false, diff --git a/crates/cli/tests/coverage/shared/doctor_tests.rs b/crates/cli/tests/coverage/shared/doctor_tests.rs index 610fe746e..e02856f63 100644 --- a/crates/cli/tests/coverage/shared/doctor_tests.rs +++ b/crates/cli/tests/coverage/shared/doctor_tests.rs @@ -483,6 +483,7 @@ fn empty_report() -> DoctorReport { upstream_auth: UpstreamAuthInfo { openai: SecretPresence::Unset, anthropic: SecretPresence::Unset, + typesafe: SecretPresence::Unset, }, plugin_configs: vec![], plugin_resolution: Check { @@ -799,11 +800,12 @@ fn format_human_reports_effective_upstream_auth_presence() { report.configuration.upstream_auth = UpstreamAuthInfo { openai: SecretPresence::Configured, anthropic: SecretPresence::Unset, + typesafe: SecretPresence::Unset, }; let rendered = format_human(&report); - assert!(rendered.contains("Upstream openai=configured anthropic=unset")); + assert!(rendered.contains("Upstream openai=configured anthropic=unset typesafe=unset")); } #[test] @@ -1772,6 +1774,8 @@ fn configuration_and_path_helpers_cover_direct_paths_and_fallbacks() { gateway: GatewayConfig { openai_auth_header: Some("Bearer openai".into()), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, ..GatewayConfig::default() }, ..ResolvedConfig::default() diff --git a/crates/cli/tests/coverage/shared/gateway_tests.rs b/crates/cli/tests/coverage/shared/gateway_tests.rs index f9a08cdd0..a6c7d5ce3 100644 --- a/crates/cli/tests/coverage/shared/gateway_tests.rs +++ b/crates/cli/tests/coverage/shared/gateway_tests.rs @@ -357,12 +357,43 @@ fn selects_provider_routes() { assert_eq!(ProviderRoute::from_path("/unsupported"), None); } +#[test] +fn selects_typesafe_system_one_route() { + for path in [ + "/systemone", + "/v1/systemone", + "/typesafe/systemone", + "/typesafe/v1/systemone", + ] { + assert_eq!( + ProviderRoute::from_path(path), + Some(ProviderRoute::TypeSafeSystemOne) + ); + } + for path in ["/typesafe/models", "/typesafe/v1/models"] { + assert_eq!( + ProviderRoute::from_path(path), + Some(ProviderRoute::TypeSafeModels) + ); + } + assert_eq!( + ProviderRoute::TypeSafeSystemOne.name(), + "typesafe.system_one" + ); + assert_eq!( + ProviderRoute::TypeSafeSystemOne.alignment_route(), + GatewayRouteKind::TypeSafeSystemOne + ); + assert_eq!(ProviderRoute::TypeSafeModels.name(), "typesafe.models"); +} + #[test] fn generation_routes_have_request_codecs_and_passthrough_routes_do_not() { for route in [ ProviderRoute::AnthropicMessages, ProviderRoute::OpenAiChatCompletions, ProviderRoute::OpenAiResponses, + ProviderRoute::TypeSafeSystemOne, ] { let codecs = codecs_for_route(route); assert!( @@ -455,6 +486,29 @@ fn dispatch_override_routes_cover_models_and_count_tokens() { "alias {alias}" ); } + for alias in [ + "typesafe_models", + "typesafe.models", + "/typesafe/models", + "/typesafe/v1/models", + ] { + assert_eq!( + ProviderRoute::from_dispatch_override(alias), + Some(ProviderRoute::TypeSafeModels), + "alias {alias}" + ); + } + for alias in [ + "typesafe_system_one", + "typesafe.system_one", + "/v1/systemone", + ] { + assert_eq!( + ProviderRoute::from_dispatch_override(alias), + Some(ProviderRoute::TypeSafeSystemOne), + "alias {alias}" + ); + } for alias in ["openai_models", "openai.models", "/models", "/v1/models"] { assert_eq!( ProviderRoute::from_dispatch_override(alias), @@ -484,6 +538,7 @@ fn provider_route_names_round_trip_through_alignment_routes() { ProviderRoute::OpenAiModels, ProviderRoute::AnthropicMessages, ProviderRoute::AnthropicCountTokens, + ProviderRoute::TypeSafeSystemOne, ] { assert_eq!( GatewayRouteKind::from_provider_name(route.name()), @@ -500,6 +555,9 @@ fn provider_routes_preserve_path_query_and_choose_upstream() { openai_auth_header: None, anthropic_base_url: "http://anthropic/".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -531,6 +589,48 @@ fn provider_routes_preserve_path_query_and_choose_upstream() { ProviderRoute::AnthropicMessages.upstream_url(&config, "/v1/messages"), "http://anthropic/v1/messages" ); + assert_eq!( + ProviderRoute::TypeSafeSystemOne.upstream_url(&config, "/v1/systemone"), + "https://api.typesafe.ai/v1/systemone" + ); + assert_eq!( + ProviderRoute::TypeSafeSystemOne.upstream_url(&config, "/typesafe/v1/systemone?trace=true"), + "https://api.typesafe.ai/v1/systemone?trace=true" + ); + assert_eq!( + ProviderRoute::TypeSafeModels.upstream_url(&config, "/typesafe/v1/models"), + "https://api.typesafe.ai/v1/models" + ); +} + +#[tokio::test] +async fn system_one_rejects_stream_field_before_forwarding() { + for stream in [json!(true), json!(false), json!(null)] { + let request = Request::builder() + .method(Method::POST) + .uri("/v1/systemone") + .body(Body::from( + json!({ + "model": "jev-latest", + "state": "candidate", + "questions": {"ok": {"type": "noul", "instructions": "Is it OK?"}}, + "stream": stream + }) + .to_string(), + )) + .unwrap(); + let error = match prepare_gateway_request( + &GatewayConfig::default(), + request, + environment_authorization(), + ) + .await + { + Ok(_) => panic!("streaming declaration must be rejected"), + Err(error) => error, + }; + assert!(error.to_string().contains("non-streaming")); + } } #[test] @@ -549,6 +649,9 @@ fn openai_upstream_url_accepts_origin_or_v1_base() { openai_auth_header: None, anthropic_base_url: "http://anthropic".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -583,6 +686,9 @@ fn anthropic_upstream_url_accepts_origin_or_v1_base() { openai_auth_header: None, anthropic_base_url: "http://anthropic".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -1010,7 +1116,7 @@ fn structured_upstream_failure_classification_matches_retry_policy() { headers.insert("content-length", HeaderValue::from_static("12")); headers.insert("retry-after", HeaderValue::from_static("3")); headers.insert("x-request-id", HeaderValue::from_static("request-123")); - for status in [408, 429, 500, 502, 503, 504] { + for status in [408, 429, 500, 502, 503, 504, 529] { let failure = http_failure( StatusCode::from_u16(status).unwrap(), &headers, @@ -1872,11 +1978,125 @@ fn injects_anthropic_x_api_key_for_anthropic_routes() { assert!(built.headers().get("authorization").is_none()); } +#[test] +fn injects_typesafe_bearer_and_bounds_retry_delay() { + let built = inject_provider_auth_with_env( + test_http_client().post("http://upstream/v1/systemone"), + ProviderRoute::TypeSafeSystemOne, + &HeaderMap::new(), + true, + None, + |key| (key == "TYPESAFE_API_KEY").then(|| "jev-secret".into()), + ) + .build() + .unwrap(); + assert_eq!( + built.headers().get(header::AUTHORIZATION).unwrap(), + "Bearer jev-secret" + ); + + let policy = crate::typesafe_retry::TypeSafeRetryPolicy::default(); + assert!(policy.should_retry_status(StatusCode::TOO_MANY_REQUESTS, 0)); + assert!(policy.should_retry_status(StatusCode::from_u16(529).unwrap(), 1)); + assert!(!policy.should_retry_status(StatusCode::TOO_MANY_REQUESTS, 2)); + let mut headers = HeaderMap::new(); + headers.insert(header::RETRY_AFTER, HeaderValue::from_static("30")); + assert_eq!(policy.delay(Some(&headers), 0), Duration::from_secs(30)); +} + +#[test] +fn typesafe_retry_policy_matches_sdk_status_and_delay_contract() { + let policy = crate::typesafe_retry::TypeSafeRetryPolicy::default(); + for status in [408, 429, 500, 529, 599] { + assert!(policy.should_retry_status(StatusCode::from_u16(status).unwrap(), 0)); + } + assert!(!policy.should_retry_status(StatusCode::BAD_REQUEST, 0)); + assert!(!policy.should_retry_status(StatusCode::TOO_MANY_REQUESTS, 2)); + + let mut headers = HeaderMap::new(); + headers.insert("retry-after-ms", HeaderValue::from_static("1250")); + assert_eq!( + policy.delay(Some(&headers), 0), + Duration::from_millis(1_250) + ); + + headers.clear(); + headers.insert(header::RETRY_AFTER, HeaderValue::from_static("1.5")); + assert_eq!( + policy.delay(Some(&headers), 0), + Duration::from_millis(1_500) + ); + + headers.insert(header::RETRY_AFTER, HeaderValue::from_static("120")); + let fallback = policy.delay(Some(&headers), 0); + assert!((375..=500).contains(&fallback.as_millis())); +} + +#[tokio::test] +async fn system_one_retries_429_and_529_then_returns_success() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + for (index, response) in [ + "HTTP/1.1 429 Too Many Requests\r\nretry-after: 0\r\ncontent-length: 0\r\nconnection: close\r\n\r\n", + "HTTP/1.1 529 Site Overloaded\r\nretry-after: 0\r\ncontent-length: 0\r\nconnection: close\r\n\r\n", + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}", + ] + .into_iter() + .enumerate() + { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 2048]; + let size = socket.read(&mut request).await.unwrap(); + let request = String::from_utf8_lossy(&request[..size]).to_ascii_lowercase(); + if index == 0 { + assert!(!request.contains("x-typesafe-retry-count:")); + } else { + assert!(request.contains(&format!("x-typesafe-retry-count: {index}"))); + } + socket.write_all(response.as_bytes()).await.unwrap(); + } + }); + + let config = GatewayConfig::default(); + let url = format!("http://{address}/v1/systemone"); + let body = Bytes::from_static(br#"{"model":"jev-latest"}"#); + let headers = HeaderMap::new(); + let method = Method::POST; + let http = test_http_client(); + let response = forward_upstream_request( + &http, + UpstreamForwardRequest { + method: &method, + url: &url, + body_bytes: &body, + headers: &headers, + effective_request: None, + forwarding: ProviderForwarding::new( + ProviderRoute::TypeSafeSystemOne, + crate::provider_auth::ProviderRequestAuthorization { + source_credential: crate::provider_auth::SourceCredentialDisposition::Absent, + allow_environment_provider_auth: false, + }, + &config, + ), + operational: None, + streaming: false, + }, + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + server.await.unwrap(); +} + #[test] fn configured_auth_headers_are_provider_specific_and_precede_environment_keys() { let config = GatewayConfig { openai_auth_header: Some("Basic openai-custom".into()), anthropic_auth_header: Some("Bearer anthropic-custom".into()), + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, ..GatewayConfig::default() }; let forwarding = ProviderForwarding::new( @@ -2243,6 +2463,9 @@ async fn passthrough_rejects_unsupported_provider_path_directly() { openai_auth_header: None, anthropic_base_url: "http://anthropic".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -2282,6 +2505,9 @@ async fn models_rejects_non_get_requests_directly() { openai_auth_header: None, anthropic_base_url: "http://anthropic".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -2687,6 +2913,9 @@ async fn models_refuses_an_unusable_named_upstream() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1:1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index 7cf007f06..d7a0883d6 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -343,6 +343,9 @@ fn test_config() -> GatewayConfig { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -3757,6 +3760,40 @@ async fn gateway_rejects_unsupported_paths() { assert_eq!(response.status(), StatusCode::NOT_FOUND); } +#[tokio::test] +async fn system_one_routes_are_registered_and_reject_streaming_before_upstream() { + let app = router(test_config()); + for path in [ + "/systemone", + "/v1/systemone", + "/typesafe/systemone", + "/typesafe/v1/systemone", + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "jev-latest", + "state": "candidate", + "questions": {"correct": {"type": "noul"}}, + "stream": false + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}"); + } +} + #[tokio::test] async fn gateway_upstream_transport_error_url_is_opaque_in_events() { const SECRET_USERNAME: &str = "secret-upstream-user"; @@ -3852,8 +3889,10 @@ async fn models_route_forwards_get_requests() { let upstream = spawn_models_upstream().await; let mut config = test_config(); config.openai_base_url = upstream.url(); + config.typesafe_base_url = upstream.url(); let app = router(config); let response = app + .clone() .oneshot( Request::builder() .method("GET") @@ -3870,6 +3909,23 @@ async fn models_route_forwards_get_requests() { let body: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(body["path"], json!("/v1/models?limit=1")); assert_eq!(body["authorization"], json!("Bearer test")); + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri("/typesafe/v1/models?limit=2") + .header("authorization", "Bearer jev-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["path"], json!("/v1/models?limit=2")); + assert_eq!(body["authorization"], json!("Bearer jev-test")); } #[tokio::test] diff --git a/crates/cli/tests/coverage/shared/session_tests.rs b/crates/cli/tests/coverage/shared/session_tests.rs index dca962f16..2d1e83e8f 100644 --- a/crates/cli/tests/coverage/shared/session_tests.rs +++ b/crates/cli/tests/coverage/shared/session_tests.rs @@ -1728,6 +1728,9 @@ async fn nests_agent_subagent_and_tool_lifecycle() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -3591,6 +3594,9 @@ async fn writes_atif_on_session_end_from_plugin_config() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -4200,6 +4206,9 @@ async fn duplicate_agent_end_does_not_overwrite_atif_with_empty_session() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -4388,6 +4397,9 @@ async fn handles_out_of_order_subagent_and_tool_end_events() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -4466,6 +4478,9 @@ async fn out_of_order_started_subagent_end_does_not_leak_scope() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -4540,6 +4555,9 @@ async fn agent_end_closes_nested_active_subagents_lifo() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -4598,6 +4616,9 @@ async fn llm_lifecycle_starts_implicit_gateway_session() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -5083,6 +5104,9 @@ async fn llm_lifecycle_uses_single_active_hook_session_when_header_is_missing() openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -5212,6 +5236,9 @@ async fn single_pending_llm_hint_claims_next_gateway_llm() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -5311,6 +5338,9 @@ async fn multiple_llm_hints_resolve_by_generation_id() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -5428,6 +5458,9 @@ async fn ambiguous_llm_hints_fall_back_to_agent_scope() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -5523,6 +5556,9 @@ async fn no_active_hint_reuses_last_llm_owner() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -7356,6 +7392,9 @@ fn session_test_config() -> GatewayConfig { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, @@ -7372,6 +7411,9 @@ async fn turn_ended_is_noop_without_active_turn_scope() { openai_auth_header: None, anthropic_base_url: "http://127.0.0.1".into(), anthropic_auth_header: None, + typesafe_base_url: "https://api.typesafe.ai/v1".into(), + typesafe_auth_header: None, + typesafe_retry: crate::typesafe_retry::TypeSafeRetryPolicy::default(), metadata: None, plugin_config: None, max_hook_payload_bytes: crate::configuration::DEFAULT_MAX_HOOK_PAYLOAD_BYTES, diff --git a/crates/core/src/api/evaluation.rs b/crates/core/src/api/evaluation.rs new file mode 100644 index 000000000..83f1acc93 --- /dev/null +++ b/crates/core/src/api/evaluation.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Provider-neutral evaluation lifecycle and managed execution API. +//! +//! Evaluation is modeled as an evaluator scope, not as chat completion. Provider adapters can +//! translate these DTOs to their wire format while subscribers and event sanitizers observe a +//! stable operation independent of any one provider. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use crate::api::scope::{ + PopScopeParams, PushScopeParams, ScopeAttributes, ScopeHandle, ScopeType, pop_scope, push_scope, +}; +use crate::api::shared::{metadata_with_otel_error, metadata_with_otel_status}; +use crate::error::{FlowError, Result}; +use crate::evaluation::{EvaluationRequest, EvaluationResponse}; +use crate::json::Json; + +/// Provider callback used by [`evaluation_execute`]. +pub type EvaluationExecutionNextFn = Arc< + dyn Fn( + EvaluationRequest, + ) -> Pin> + Send + 'static>> + + Send + + Sync, +>; + +/// Runtime-owned handle identifying an active evaluator scope. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvaluationHandle { + scope: ScopeHandle, +} + +impl EvaluationHandle { + /// Return the underlying scope handle for parenting and advanced runtime integration. + pub fn scope(&self) -> &ScopeHandle { + &self.scope + } +} + +/// Builder parameters for [`evaluation_call`]. +#[derive(TypedBuilder)] +#[builder(field_defaults(setter(strip_option(ignore_invalid, fallback_suffix = "_opt"))))] +pub struct EvaluationCallParams<'a> { + /// Logical provider or evaluator name recorded on lifecycle events. + pub name: &'a str, + /// Provider-neutral request recorded as evaluator input. + pub request: &'a EvaluationRequest, + /// Optional explicit parent scope. + #[builder(default)] + pub parent: Option<&'a ScopeHandle>, + /// Scope behavior flags. + #[builder(default = ScopeAttributes::empty())] + pub attributes: ScopeAttributes, + /// Optional application payload stored on the handle. + #[builder(default)] + pub data: Option, + /// Optional metadata recorded on the start event. + #[builder(default)] + pub metadata: Option, +} + +/// Builder parameters for [`evaluation_call_end`]. +#[derive(TypedBuilder)] +#[builder(field_defaults(setter(strip_option(ignore_invalid, fallback_suffix = "_opt"))))] +pub struct EvaluationCallEndParams<'a> { + /// Active evaluation handle to close. + pub handle: &'a EvaluationHandle, + /// Provider-neutral response recorded as evaluator output. + pub response: &'a EvaluationResponse, + /// Optional metadata merged onto the end event. + #[builder(default)] + pub metadata: Option, +} + +/// Builder parameters for [`evaluation_execute`]. +#[derive(TypedBuilder)] +#[builder(field_defaults(setter(strip_option(ignore_invalid, fallback_suffix = "_opt"))))] +pub struct EvaluationExecuteParams { + /// Logical provider or evaluator name recorded on lifecycle events. + #[builder(setter(into))] + pub name: String, + /// Provider-neutral request supplied to the provider callback. + pub request: EvaluationRequest, + /// Provider callback or execution continuation. + pub func: EvaluationExecutionNextFn, + /// Optional explicit parent scope. + #[builder(default)] + pub parent: Option, + /// Scope behavior flags. + #[builder(default = ScopeAttributes::empty())] + pub attributes: ScopeAttributes, + /// Optional application payload stored on the handle. + #[builder(default)] + pub data: Option, + /// Optional metadata recorded on lifecycle events. + #[builder(default)] + pub metadata: Option, +} + +/// Start a manual provider-neutral evaluation lifecycle. +pub fn evaluation_call(params: EvaluationCallParams<'_>) -> Result { + let input = serde_json::to_value(params.request) + .map_err(|error| FlowError::Internal(error.to_string()))?; + let scope = push_scope( + PushScopeParams::builder() + .name(params.name) + .scope_type(ScopeType::Evaluator) + .parent_opt(params.parent) + .attributes(params.attributes) + .data_opt(params.data) + .metadata_opt(params.metadata) + .input(input) + .build(), + )?; + Ok(EvaluationHandle { scope }) +} + +/// Finish a manual provider-neutral evaluation lifecycle. +pub fn evaluation_call_end(params: EvaluationCallEndParams<'_>) -> Result<()> { + let output = serde_json::to_value(params.response) + .map_err(|error| FlowError::Internal(error.to_string()))?; + pop_scope( + PopScopeParams::builder() + .handle_uuid(¶ms.handle.scope.uuid) + .output(output) + .metadata_opt(params.metadata) + .build(), + ) +} + +/// Execute a provider-neutral evaluation callback inside a complete evaluator lifecycle. +/// +/// Event sanitizers run on both lifecycle events. Success and failure are represented with the +/// same OpenTelemetry status metadata used by the LLM and tool managed-execution APIs. +pub async fn evaluation_execute(params: EvaluationExecuteParams) -> Result { + let handle = evaluation_call( + EvaluationCallParams::builder() + .name(¶ms.name) + .request(¶ms.request) + .parent_opt(params.parent.as_ref()) + .attributes(params.attributes) + .data_opt(params.data) + .metadata_opt(params.metadata.clone()) + .build(), + )?; + match (params.func)(params.request).await { + Ok(response) => { + evaluation_call_end( + EvaluationCallEndParams::builder() + .handle(&handle) + .response(&response) + .metadata_opt(metadata_with_otel_status(params.metadata, "OK", None)) + .build(), + )?; + Ok(response) + } + Err(error) => { + pop_scope( + PopScopeParams::builder() + .handle_uuid(&handle.scope.uuid) + .metadata_opt(metadata_with_otel_error(params.metadata, &error)) + .build(), + )?; + Err(error) + } + } +} + +#[cfg(test)] +#[path = "../../tests/unit/evaluation_api_tests.rs"] +mod tests; diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs index 21b5b83b3..c74f7fe7f 100644 --- a/crates/core/src/api/mod.rs +++ b/crates/core/src/api/mod.rs @@ -3,6 +3,8 @@ //! Public API for the NeMo Relay runtime. +/// Provider-neutral evaluator lifecycle and managed execution entry points. +pub mod evaluation; /// Lifecycle event types and builder-backed event constructors. pub mod event; /// LLM lifecycle helpers and managed execution entry points. diff --git a/crates/core/src/codec/mod.rs b/crates/core/src/codec/mod.rs index dc5ca237e..fa24bdbb7 100644 --- a/crates/core/src/codec/mod.rs +++ b/crates/core/src/codec/mod.rs @@ -26,6 +26,7 @@ pub mod resolve; pub mod response; pub mod streaming; pub mod traits; +pub mod typesafe_system_one; use nemo_relay_types::Json; diff --git a/crates/core/src/codec/resolve.rs b/crates/core/src/codec/resolve.rs index 30a585520..e5ac0083f 100644 --- a/crates/core/src/codec/resolve.rs +++ b/crates/core/src/codec/resolve.rs @@ -14,7 +14,10 @@ use super::request::AnnotatedLlmRequest; use super::response::AnnotatedLlmResponse; use super::streaming::StreamingCodec; use super::traits::{LlmCodec, LlmResponseCodec}; -use super::{anthropic, gemini_generate_content, oci_genai, openai_chat, openai_responses}; +use super::{ + anthropic, gemini_generate_content, oci_genai, openai_chat, openai_responses, + typesafe_system_one, +}; /// A built-in provider request/response surface. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -29,6 +32,8 @@ pub enum ProviderSurface { OCIGenAI, /// Gemini generateContent. GeminiGenerateContent, + /// TypeSafe System One evaluation. + TypeSafeSystemOne, } /// Request shape detector; the optional `&str` is a provider hint a codec may use @@ -68,6 +73,7 @@ pub(crate) struct ProviderSurfaceDescriptor { /// surface it could shadow. Response detection requires exactly one match /// before decoding. pub(crate) static BUILTIN_PROVIDER_SURFACES: &[ProviderSurfaceDescriptor] = &[ + typesafe_system_one::PROVIDER_SURFACE, openai_responses::PROVIDER_SURFACE, anthropic::PROVIDER_SURFACE, // OCI GenAI must precede OpenAI Chat: a bare OCI GENERIC chatRequest body @@ -80,7 +86,8 @@ pub(crate) static BUILTIN_PROVIDER_SURFACES: &[ProviderSurfaceDescriptor] = &[ /// Detect the request surface from a raw request body by top-level key. /// -/// Priority: OpenAI Responses (`input`/`instructions`) > Anthropic Messages +/// Priority: TypeSafe System One (`questions`/`criteria`) > OpenAI Responses +/// (`input`/`instructions`) > Anthropic Messages /// (`system`) > OpenAI Chat (`messages`) > Gemini generateContent (`contents`). /// `None` when no key matches or `body` is not an object. This is a best-effort heuristic: an /// Anthropic request that omits the optional top-level `system` is @@ -163,6 +170,7 @@ fn descriptor_for(surface: ProviderSurface) -> &'static ProviderSurfaceDescripto ProviderSurface::AnthropicMessages => &anthropic::PROVIDER_SURFACE, ProviderSurface::OCIGenAI => &oci_genai::PROVIDER_SURFACE, ProviderSurface::GeminiGenerateContent => &gemini_generate_content::PROVIDER_SURFACE, + ProviderSurface::TypeSafeSystemOne => &typesafe_system_one::PROVIDER_SURFACE, } } diff --git a/crates/core/src/codec/typesafe_system_one.rs b/crates/core/src/codec/typesafe_system_one.rs new file mode 100644 index 000000000..b4571aa5e --- /dev/null +++ b/crates/core/src/codec/typesafe_system_one.rs @@ -0,0 +1,735 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Built-in codec for TypeSafe AI's System One evaluation API. +//! +//! System One is an evaluation surface, not a chat or text-generation API. +//! The codec therefore leaves messages, tools, text output, and finish reasons +//! empty and carries provider-neutral evaluation data in the annotated +//! request/response `custom` envelopes. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::api::llm::LlmRequest; +use crate::api::runtime::{BuiltinLlmCodec, LlmCodecIdentity}; +use crate::error::{FlowError, Result}; +use crate::json::Json; +use nemo_relay_types::evaluation::{ + BooleanCriteria, EvaluationAnswer, EvaluationQuestion, EvaluationRequest, EvaluationResponse, + EvaluationUsage, +}; + +use super::request::{AnnotatedLlmRequest, ApiSpecificRequest}; +use super::resolve::{ProviderSurface, ProviderSurfaceDescriptor}; +use super::response::{ + AnnotatedLlmResponse, ApiSpecificResponse, Usage, estimate_cost_for_provider, +}; +use super::traits::{LlmCodec, LlmResponseCodec}; + +const API_NAME: &str = "typesafe.system_one"; +const PROVIDER: &str = "typesafe"; +const OPERATION: &str = "system_one"; +const MODELED_REQUEST_KEYS: &[&str] = &["model", "state", "questions"]; +const MODELED_RESPONSE_KEYS: &[&str] = &["model", "answers", "usage"]; + +/// Built-in codec for `POST /v1/systemone`. +pub struct TypeSafeSystemOneCodec; + +pub(crate) const PROVIDER_SURFACE: ProviderSurfaceDescriptor = ProviderSurfaceDescriptor { + surface: ProviderSurface::TypeSafeSystemOne, + detect_request: |obj, _hint| { + obj.get("state").is_some_and(is_entry_value) + && obj.get("questions").is_some_and(Value::is_object) + }, + detect_response: |obj| { + obj.get("answers").is_some_and(Value::is_object) + && obj.get("model").is_some_and(Value::is_string) + }, + decode_request: |request| TypeSafeSystemOneCodec.decode(request), + decode_response: |raw| TypeSafeSystemOneCodec.decode_response(raw), + codec_name: "typesafe_system_one", + request_codec: || std::sync::Arc::new(TypeSafeSystemOneCodec), + response_codec: || std::sync::Arc::new(TypeSafeSystemOneCodec), + streaming_codec: || Box::new(UnsupportedSystemOneStreamingCodec), +}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct EvaluationEnvelope { + provider: String, + operation: String, + value: T, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +enum WireQuestion { + #[serde(rename = "noul")] + Noul { + #[serde(default)] + instructions: Json, + #[serde(skip_serializing_if = "Option::is_none")] + criteria: Option, + }, + #[serde(rename = "choice")] + Choice { + #[serde(default)] + instructions: Json, + criteria: BTreeMap, + }, + #[serde(rename = "score")] + Score { + #[serde(default)] + instructions: Json, + criteria: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +enum WireAnswer { + #[serde(rename = "noul")] + Noul { noul: f64 }, + #[serde(rename = "choice")] + Choice { + choice: String, + probabilities: BTreeMap, + confidence: f64, + }, + #[serde(rename = "score")] + Score { + score: f64, + legend: BTreeMap, + probabilities: BTreeMap, + confidence: f64, + }, +} + +#[derive(Debug, Deserialize)] +struct WireResponse { + model: String, + answers: BTreeMap, + usage: WireUsage, + #[serde(flatten)] + extra: Map, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +struct WireUsage { + #[serde(default)] + billing_units: Option, + #[serde(default)] + input_tokens: Option, + #[serde(default)] + output_tokens: Option, +} + +impl From for EvaluationQuestion { + fn from(value: WireQuestion) -> Self { + match value { + WireQuestion::Noul { + instructions, + criteria, + } => Self::Boolean { + instructions, + criteria, + }, + WireQuestion::Choice { + instructions, + criteria, + } => Self::Choice { + instructions, + criteria, + }, + WireQuestion::Score { + instructions, + criteria, + } => Self::Score { + instructions, + criteria, + }, + } + } +} + +impl From<&EvaluationQuestion> for WireQuestion { + fn from(value: &EvaluationQuestion) -> Self { + match value { + EvaluationQuestion::Boolean { + instructions, + criteria, + } => Self::Noul { + instructions: instructions.clone(), + criteria: criteria.clone(), + }, + EvaluationQuestion::Choice { + instructions, + criteria, + } => Self::Choice { + instructions: instructions.clone(), + criteria: criteria.clone(), + }, + EvaluationQuestion::Score { + instructions, + criteria, + } => Self::Score { + instructions: instructions.clone(), + criteria: criteria.clone(), + }, + } + } +} + +impl From for EvaluationAnswer { + fn from(value: WireAnswer) -> Self { + match value { + WireAnswer::Noul { noul } => Self::Boolean { probability: noul }, + WireAnswer::Choice { + choice, + probabilities, + confidence, + } => Self::Choice { + choice, + probabilities, + confidence: Some(confidence), + }, + WireAnswer::Score { + score, + legend, + probabilities, + confidence, + } => Self::Score { + score, + legend, + probabilities, + confidence: Some(confidence), + }, + } + } +} + +fn decode_evaluation_request(obj: &Map) -> Result { + if obj.contains_key("stream") { + return Err(FlowError::InvalidArgument( + "TypeSafe System One does not support streaming; remove the stream field".into(), + )); + } + let model = required_non_empty_string(obj, "model", "TypeSafe System One request")?; + let state = obj + .get("state") + .filter(|value| is_entry_value(value)) + .cloned() + .ok_or_else(|| { + FlowError::InvalidArgument( + "TypeSafe System One request state must be a string, object, array, or null".into(), + ) + })?; + let questions_value = obj + .get("questions") + .and_then(Value::as_object) + .ok_or_else(|| { + FlowError::InvalidArgument( + "TypeSafe System One request questions must be an object".into(), + ) + })?; + if questions_value.is_empty() { + return Err(FlowError::InvalidArgument( + "TypeSafe System One request questions must not be empty".into(), + )); + } + let mut questions = BTreeMap::new(); + for (id, value) in questions_value { + if id.trim().is_empty() { + return Err(FlowError::InvalidArgument( + "TypeSafe System One question IDs must not be empty".into(), + )); + } + let wire: WireQuestion = serde_json::from_value(value.clone()).map_err(|error| { + FlowError::InvalidArgument(format!( + "TypeSafe System One question '{id}' is invalid: {error}" + )) + })?; + let question: EvaluationQuestion = wire.into(); + validate_question(id, &question)?; + questions.insert(id.clone(), question); + } + Ok(EvaluationRequest { + model, + state, + questions, + }) +} + +fn validate_question(id: &str, question: &EvaluationQuestion) -> Result<()> { + let instructions = match question { + EvaluationQuestion::Boolean { instructions, .. } + | EvaluationQuestion::Choice { instructions, .. } + | EvaluationQuestion::Score { instructions, .. } => instructions, + }; + if !is_entry_value(instructions) { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One question '{id}' instructions must be a string, object, array, or null" + ))); + } + match question { + EvaluationQuestion::Boolean { + criteria: Some(criteria), + .. + } if criteria + .true_description + .iter() + .chain(criteria.false_description.iter()) + .any(|value| !is_entry_value(value)) => + { + Err(FlowError::InvalidArgument(format!( + "TypeSafe System One boolean question '{id}' criteria must contain JSON entry values" + ))) + } + EvaluationQuestion::Choice { criteria, .. } + if criteria.values().any(|value| !is_entry_value(value)) => + { + Err(FlowError::InvalidArgument(format!( + "TypeSafe System One choice question '{id}' criteria must contain JSON entry values" + ))) + } + EvaluationQuestion::Score { criteria, .. } if criteria.is_empty() => { + Err(FlowError::InvalidArgument(format!( + "TypeSafe System One score question '{id}' needs at least one criterion" + ))) + } + EvaluationQuestion::Score { criteria, .. } + if criteria.iter().any(|value| !is_entry_value(value)) => + { + Err(FlowError::InvalidArgument(format!( + "TypeSafe System One score question '{id}' criteria must contain JSON entry values" + ))) + } + _ => Ok(()), + } +} + +fn is_entry_value(value: &Json) -> bool { + value.is_string() || value.is_object() || value.is_array() || value.is_null() +} + +fn required_non_empty_string(obj: &Map, key: &str, surface: &str) -> Result { + obj.get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + FlowError::InvalidArgument(format!("{surface} {key} must be a non-empty string")) + }) +} + +fn request_envelope(request: &EvaluationRequest) -> Result { + serde_json::to_value(EvaluationEnvelope { + provider: PROVIDER.to_string(), + operation: OPERATION.to_string(), + value: request.clone(), + }) + .map_err(|error| FlowError::Internal(format!("evaluation request encode: {error}"))) +} + +/// Extract the provider-neutral evaluation value from a decoded System One request. +pub fn evaluation_request(annotated: &AnnotatedLlmRequest) -> Result { + let Some(ApiSpecificRequest::Custom { api_name, data }) = annotated.api_specific.as_ref() + else { + return Err(FlowError::InvalidArgument( + "TypeSafe System One annotations require api_specific custom evaluation data".into(), + )); + }; + if api_name != API_NAME { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One api_specific provider mismatch: expected {API_NAME}, got {api_name}" + ))); + } + let envelope: EvaluationEnvelope = serde_json::from_value(data.clone()) + .map_err(|error| { + FlowError::InvalidArgument(format!( + "TypeSafe System One evaluation annotation is invalid: {error}" + )) + })?; + if envelope.provider != PROVIDER || envelope.operation != OPERATION { + return Err(FlowError::InvalidArgument( + "TypeSafe System One evaluation annotation has the wrong provider or operation".into(), + )); + } + Ok(envelope.value) +} + +/// Extract the provider-neutral evaluation value from a decoded System One response. +pub fn evaluation_response(annotated: &AnnotatedLlmResponse) -> Result { + let Some(ApiSpecificResponse::Custom { api_name, data }) = annotated.api_specific.as_ref() + else { + return Err(FlowError::InvalidArgument( + "TypeSafe System One annotations require api_specific custom evaluation data".into(), + )); + }; + if api_name != API_NAME { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One api_specific provider mismatch: expected {API_NAME}, got {api_name}" + ))); + } + let envelope: EvaluationEnvelope = serde_json::from_value(data.clone()) + .map_err(|error| { + FlowError::InvalidArgument(format!( + "TypeSafe System One evaluation annotation is invalid: {error}" + )) + })?; + if envelope.provider != PROVIDER || envelope.operation != OPERATION { + return Err(FlowError::InvalidArgument( + "TypeSafe System One evaluation annotation has the wrong provider or operation".into(), + )); + } + Ok(envelope.value) +} + +fn encode_question(question: &EvaluationQuestion) -> Result { + serde_json::to_value(WireQuestion::from(question)) + .map_err(|error| FlowError::Internal(format!("System One question encode: {error}"))) +} + +fn patch_questions( + obj: &mut Map, + edited: &BTreeMap, + baseline: &BTreeMap, +) -> Result<()> { + let original = obj + .get("questions") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + let mut patched = Map::new(); + for (id, question) in edited { + validate_question(id, question)?; + if baseline.get(id) == Some(question) + && let Some(original_question) = original.get(id) + { + patched.insert(id.clone(), original_question.clone()); + continue; + } + let mut encoded = encode_question(question)?; + if let (Some(encoded), Some(original)) = ( + encoded.as_object_mut(), + original.get(id).and_then(Value::as_object), + ) { + // Boolean criteria have two modeled keys, so other keys are provider extensions. + // Choice criteria are themselves the option map: removed options must stay removed. + if matches!(question, EvaluationQuestion::Boolean { .. }) + && let (Some(encoded_criteria), Some(original_criteria)) = ( + encoded.get_mut("criteria").and_then(Value::as_object_mut), + original.get("criteria").and_then(Value::as_object), + ) + { + for (key, value) in original_criteria { + if !matches!(key.as_str(), "true" | "false") { + encoded_criteria + .entry(key.clone()) + .or_insert_with(|| value.clone()); + } + } + } + for (key, value) in original { + if !matches!(key.as_str(), "type" | "instructions" | "criteria") { + encoded.entry(key.clone()).or_insert_with(|| value.clone()); + } + } + } + patched.insert(id.clone(), encoded); + } + obj.insert("questions".into(), Json::Object(patched)); + Ok(()) +} + +fn patch_extra_fields( + obj: &mut Map, + baseline: &Map, + edited: &Map, +) { + for key in baseline.keys() { + if !edited.contains_key(key) { + obj.remove(key); + } + } + for (key, value) in edited { + if baseline.get(key) != Some(value) { + obj.insert(key.clone(), value.clone()); + } + } +} + +fn validate_probability(value: f64, field: &str) -> Result<()> { + if value.is_finite() && (0.0..=1.0).contains(&value) { + Ok(()) + } else { + Err(FlowError::InvalidArgument(format!( + "TypeSafe System One response {field} must be between 0 and 1" + ))) + } +} + +fn validate_answer(id: &str, answer: &EvaluationAnswer) -> Result<()> { + match answer { + EvaluationAnswer::Boolean { probability } => { + validate_probability(*probability, &format!("answer '{id}' probability")) + } + EvaluationAnswer::Choice { + choice, + probabilities, + confidence, + } => { + if !probabilities.contains_key(choice) { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One choice answer '{id}' selected an option missing from probabilities" + ))); + } + for (option, probability) in probabilities { + validate_probability( + *probability, + &format!("answer '{id}' probability for '{option}'"), + )?; + } + if let Some(confidence) = confidence { + validate_probability(*confidence, &format!("answer '{id}' confidence"))?; + } + Ok(()) + } + EvaluationAnswer::Score { + probabilities, + confidence, + .. + } => { + for (level, probability) in probabilities { + validate_probability( + *probability, + &format!("answer '{id}' probability for level '{level}'"), + )?; + } + if let Some(confidence) = confidence { + validate_probability(*confidence, &format!("answer '{id}' confidence"))?; + } + Ok(()) + } + } +} + +impl LlmCodec for TypeSafeSystemOneCodec { + fn codec_identity(&self) -> LlmCodecIdentity { + LlmCodecIdentity::BuiltIn(BuiltinLlmCodec::TypeSafeSystemOne) + } + + fn decode(&self, request: &LlmRequest) -> Result { + let obj = request.content.as_object().ok_or_else(|| { + FlowError::InvalidArgument("TypeSafe System One request must be an object".into()) + })?; + let evaluation = decode_evaluation_request(obj)?; + let extra = obj + .iter() + .filter(|(key, _)| !MODELED_REQUEST_KEYS.contains(&key.as_str())) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + Ok(AnnotatedLlmRequest { + model: Some(evaluation.model.clone()), + api_specific: Some(ApiSpecificRequest::Custom { + api_name: API_NAME.into(), + data: request_envelope(&evaluation)?, + }), + extra, + ..AnnotatedLlmRequest::default() + }) + } + + fn encode(&self, annotated: &AnnotatedLlmRequest, original: &LlmRequest) -> Result { + let baseline = self.decode(original)?; + validate_non_evaluation_fields(annotated, &baseline)?; + let mut edited = evaluation_request(annotated)?; + let baseline_evaluation = evaluation_request(&baseline)?; + + let annotation_model_changed = edited.model != baseline_evaluation.model; + let top_level_model_changed = annotated.model != baseline.model; + if annotation_model_changed + && top_level_model_changed + && annotated.model.as_ref() != Some(&edited.model) + { + return Err(FlowError::InvalidArgument( + "TypeSafe System One model was changed inconsistently in model and api_specific" + .into(), + )); + } + if top_level_model_changed { + edited.model = annotated.model.clone().ok_or_else(|| { + FlowError::InvalidArgument("TypeSafe System One model cannot be removed".into()) + })?; + } + if edited.model.trim().is_empty() { + return Err(FlowError::InvalidArgument( + "TypeSafe System One model must not be empty".into(), + )); + } + + let mut content = original.content.clone(); + let obj = content.as_object_mut().ok_or_else(|| { + FlowError::InvalidArgument("TypeSafe System One request must be an object".into()) + })?; + if edited.model != baseline_evaluation.model { + obj.insert("model".into(), Json::String(edited.model.clone())); + } + if edited.state != baseline_evaluation.state { + if !is_entry_value(&edited.state) { + return Err(FlowError::InvalidArgument( + "TypeSafe System One state must be a string, object, array, or null".into(), + )); + } + obj.insert("state".into(), edited.state.clone()); + } + if edited.questions != baseline_evaluation.questions { + if edited.questions.is_empty() { + return Err(FlowError::InvalidArgument( + "TypeSafe System One questions must not be empty".into(), + )); + } + patch_questions(obj, &edited.questions, &baseline_evaluation.questions)?; + } + patch_extra_fields(obj, &baseline.extra, &annotated.extra); + Ok(LlmRequest { + headers: original.headers.clone(), + content, + }) + } +} + +fn validate_non_evaluation_fields( + annotated: &AnnotatedLlmRequest, + baseline: &AnnotatedLlmRequest, +) -> Result<()> { + if let Some(key) = annotated + .extra + .keys() + .find(|key| MODELED_REQUEST_KEYS.contains(&key.as_str())) + { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One modeled field '{key}' cannot be edited through extra" + ))); + } + macro_rules! reject_if_changed { + ($field:ident) => { + if annotated.$field != baseline.$field { + return Err(FlowError::InvalidArgument(format!( + "TypeSafe System One does not support annotated field '{}'", + stringify!($field) + ))); + } + }; + } + reject_if_changed!(messages); + reject_if_changed!(instructions); + reject_if_changed!(params); + reject_if_changed!(tools); + reject_if_changed!(tool_choice); + reject_if_changed!(store); + reject_if_changed!(previous_response_id); + reject_if_changed!(truncation); + reject_if_changed!(reasoning); + reject_if_changed!(include); + reject_if_changed!(user); + reject_if_changed!(metadata); + reject_if_changed!(service_tier); + reject_if_changed!(parallel_tool_calls); + reject_if_changed!(max_output_tokens); + reject_if_changed!(max_tool_calls); + reject_if_changed!(top_logprobs); + reject_if_changed!(stream); + Ok(()) +} + +impl LlmResponseCodec for TypeSafeSystemOneCodec { + fn codec_identity(&self) -> LlmCodecIdentity { + LlmCodecIdentity::BuiltIn(BuiltinLlmCodec::TypeSafeSystemOne) + } + + fn decode_response(&self, response: &Json) -> Result { + let raw: WireResponse = serde_json::from_value(response.clone()).map_err(|error| { + FlowError::InvalidArgument(format!("TypeSafe System One response is invalid: {error}")) + })?; + if raw.model.trim().is_empty() { + return Err(FlowError::InvalidArgument( + "TypeSafe System One response model must not be empty".into(), + )); + } + let answers = raw + .answers + .into_iter() + .map(|(id, answer)| { + let answer: EvaluationAnswer = answer.into(); + validate_answer(&id, &answer)?; + Ok((id, answer)) + }) + .collect::>>()?; + let evaluation = EvaluationResponse { + model: raw.model.clone(), + answers, + usage: Some(EvaluationUsage { + billing_units: raw.usage.billing_units, + input_tokens: raw.usage.input_tokens, + output_tokens: raw.usage.output_tokens, + }), + }; + let mut usage = Usage { + prompt_tokens: raw.usage.input_tokens, + completion_tokens: raw.usage.output_tokens, + total_tokens: raw + .usage + .input_tokens + .zip(raw.usage.output_tokens) + .and_then(|(input, output)| input.checked_add(output)), + ..Usage::default() + }; + usage.cost = estimate_cost_for_provider(Some(PROVIDER), &raw.model, &usage); + let data = serde_json::to_value(EvaluationEnvelope { + provider: PROVIDER.to_string(), + operation: OPERATION.to_string(), + value: evaluation, + }) + .map_err(|error| FlowError::Internal(format!("evaluation response encode: {error}")))?; + Ok(AnnotatedLlmResponse { + model: Some(raw.model), + usage: Some(usage), + api_specific: Some(ApiSpecificResponse::Custom { + api_name: API_NAME.into(), + data, + }), + extra: raw + .extra + .into_iter() + .filter(|(key, _)| !MODELED_RESPONSE_KEYS.contains(&key.as_str())) + .collect(), + ..AnnotatedLlmResponse::default() + }) + } +} + +/// Defensive streaming placeholder. Requests declaring streaming are rejected +/// by `decode` before a provider continuation can be opened. +struct UnsupportedSystemOneStreamingCodec; + +impl super::streaming::StreamingCodec for UnsupportedSystemOneStreamingCodec { + fn collector(&self) -> crate::api::runtime::LlmCollectorFn { + Box::new(|_| { + Err(FlowError::InvalidArgument( + "TypeSafe System One does not support streaming".into(), + )) + }) + } + + fn finalizer(&self) -> crate::api::runtime::LlmFinalizerFn { + Box::new(|| Json::Null) + } +} + +#[cfg(test)] +#[path = "../../tests/unit/codec/typesafe_system_one_tests.rs"] +mod tests; diff --git a/crates/core/src/evaluation.rs b/crates/core/src/evaluation.rs new file mode 100644 index 000000000..0883c5634 --- /dev/null +++ b/crates/core/src/evaluation.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Provider-neutral decision and evaluation types. + +pub use nemo_relay_types::evaluation::*; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 12444ee56..64e8bfda6 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,6 +23,7 @@ //! //! - [`api::scope::push_scope`] / [`api::scope::pop_scope`] create nested execution scopes. //! - [`api::tool::tool_call_execute`] runs a complete tool middleware pipeline. +//! - [`api::evaluation::evaluation_execute`] runs a provider-neutral evaluator lifecycle. //! - [`api::llm::llm_call_execute`] and [`api::llm::llm_stream_call_execute`] run non-streaming //! and streaming LLM middleware pipelines. //! - [`api::registry`] exposes global and scope-local middleware registration APIs. @@ -59,6 +60,7 @@ pub mod codec; pub mod config_editor; mod context; pub mod error; +pub mod evaluation; pub mod json; pub mod logging; pub mod observability; diff --git a/crates/core/src/observability/otel_genai.rs b/crates/core/src/observability/otel_genai.rs index f0832640f..51e802462 100644 --- a/crates/core/src/observability/otel_genai.rs +++ b/crates/core/src/observability/otel_genai.rs @@ -19,6 +19,7 @@ use serde_json::{Map, Value}; const OPERATION_CHAT: &str = "chat"; const OPERATION_EMBEDDINGS: &str = "embeddings"; +const OPERATION_EVALUATE: &str = "evaluate"; const OPERATION_EXECUTE_TOOL: &str = "execute_tool"; const OPERATION_GENERATE_CONTENT: &str = "generate_content"; const OPERATION_INVOKE_AGENT: &str = "invoke_agent"; @@ -35,6 +36,14 @@ const GEN_AI_SYSTEM_INSTRUCTIONS: &str = "gen_ai.system_instructions"; const GEN_AI_RETRIEVAL_TOP_K: &str = "gen_ai.retrieval.top_k"; const GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS: &str = "gen_ai.usage.cache_creation.input_tokens"; const GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS: &str = "gen_ai.usage.cache_read.input_tokens"; +const EVALUATION_API_NAME: &str = "typesafe.system_one"; +const EVALUATION_PROVIDER: &str = "nemo_relay.evaluation.provider"; +const EVALUATION_OPERATION: &str = "nemo_relay.evaluation.operation"; +const EVALUATION_QUESTION_COUNT: &str = "nemo_relay.evaluation.question_count"; +const EVALUATION_ANSWER_COUNT: &str = "nemo_relay.evaluation.answer_count"; +const EVALUATION_BOOLEAN_ANSWER_COUNT: &str = "nemo_relay.evaluation.boolean_answer_count"; +const EVALUATION_CHOICE_ANSWER_COUNT: &str = "nemo_relay.evaluation.choice_answer_count"; +const EVALUATION_SCORE_ANSWER_COUNT: &str = "nemo_relay.evaluation.score_answer_count"; fn has_gen_ai_semantics(event: &Event) -> bool { matches!( @@ -45,6 +54,7 @@ fn has_gen_ai_semantics(event: &Event) -> bool { | ScopeType::Tool | ScopeType::Embedder | ScopeType::Retriever + | ScopeType::Evaluator ) ) } @@ -58,7 +68,7 @@ pub(super) fn span_name(event: &Event) -> String { Some(ScopeType::Agent) => Some(agent_name(event)), Some(ScopeType::Tool) => Some(tool_name(event)), Some(ScopeType::Retriever) => data_source_id(event), - Some(ScopeType::Llm | ScopeType::Embedder) => request_model(event), + Some(ScopeType::Llm | ScopeType::Embedder | ScopeType::Evaluator) => request_model(event), _ => None, }; qualifier.filter(|value| !value.is_empty()).map_or_else( @@ -70,7 +80,9 @@ pub(super) fn span_name(event: &Event) -> String { pub(super) fn span_kind(event: &Event) -> SpanKind { match event.scope_type() { Some(ScopeType::Agent | ScopeType::Tool) => SpanKind::Internal, - Some(ScopeType::Llm | ScopeType::Embedder | ScopeType::Retriever) => SpanKind::Client, + Some( + ScopeType::Llm | ScopeType::Embedder | ScopeType::Retriever | ScopeType::Evaluator, + ) => SpanKind::Client, _ => SpanKind::Internal, } } @@ -108,6 +120,10 @@ pub(super) fn start_attributes(event: &Event) -> Vec { push_provider_and_server_attributes(&mut attributes, event); push_model_attribute(&mut attributes, event); } + Some(ScopeType::Evaluator) => { + push_provider_and_server_attributes(&mut attributes, event); + push_evaluator_request_attributes(&mut attributes, event); + } _ => {} } attributes @@ -127,6 +143,7 @@ pub(super) fn end_attributes(event: &Event) -> Vec { push_tool_content(&mut attributes, "gen_ai.tool.call.result", event.output()); } } + Some(ScopeType::Evaluator) => push_evaluator_response_attributes(&mut attributes, event), _ => {} } attributes @@ -155,12 +172,87 @@ fn operation_name(event: &Event) -> &'static str { Some(ScopeType::Tool) => OPERATION_EXECUTE_TOOL, Some(ScopeType::Embedder) => OPERATION_EMBEDDINGS, Some(ScopeType::Retriever) => OPERATION_RETRIEVAL, + Some(ScopeType::Evaluator) => OPERATION_EVALUATE, Some(ScopeType::Llm) => llm_operation_name(event), _ => OPERATION_CHAT, } } +fn push_evaluator_request_attributes(attributes: &mut Vec, event: &Event) { + push_model_attribute(attributes, event); + if let Some(count) = event + .input() + .and_then(|input| input.get("questions")) + .and_then(Value::as_object) + .and_then(|questions| i64::try_from(questions.len()).ok()) + { + attributes.push(KeyValue::new(EVALUATION_QUESTION_COUNT, count)); + } +} + +fn push_evaluator_response_attributes(attributes: &mut Vec, event: &Event) { + let Some(output) = event.output() else { + return; + }; + if let Some(model) = output.get("model").and_then(Value::as_str) { + attributes.push(KeyValue::new("gen_ai.response.model", model.to_owned())); + } + if let Some(usage) = output.get("usage").and_then(Value::as_object) { + if let Some(value) = usage + .get("input_tokens") + .and_then(Value::as_u64) + .and_then(to_i64) + { + attributes.push(KeyValue::new(semconv::GEN_AI_USAGE_INPUT_TOKENS, value)); + } + if let Some(value) = usage + .get("output_tokens") + .and_then(Value::as_u64) + .and_then(to_i64) + { + attributes.push(KeyValue::new("gen_ai.usage.output_tokens", value)); + } + if let Some(value) = usage + .get("billing_units") + .and_then(Value::as_u64) + .and_then(to_i64) + { + attributes.push(KeyValue::new("nemo_relay.evaluation.billing_units", value)); + } + } + if let Some(answers) = output.get("answers").and_then(Value::as_object) { + push_evaluation_answer_counts(attributes, answers); + } +} + +fn push_evaluation_answer_counts(attributes: &mut Vec, answers: &Map) { + if let Ok(count) = i64::try_from(answers.len()) { + attributes.push(KeyValue::new(EVALUATION_ANSWER_COUNT, count)); + } + for (kind, attribute) in [ + ("boolean", EVALUATION_BOOLEAN_ANSWER_COUNT), + ("choice", EVALUATION_CHOICE_ANSWER_COUNT), + ("score", EVALUATION_SCORE_ANSWER_COUNT), + ] { + let count = answers + .values() + .filter(|answer| answer.get("type").and_then(Value::as_str) == Some(kind)) + .count(); + if let Ok(count) = i64::try_from(count) { + attributes.push(KeyValue::new(attribute, count)); + } + } +} + fn llm_operation_name(event: &Event) -> &'static str { + if event.normalized_llm_request().is_some_and(|request| { + matches!( + request.api_specific.as_ref(), + Some(ApiSpecificRequest::Custom { api_name, .. }) if api_name == EVALUATION_API_NAME + ) + }) { + return OPERATION_EVALUATE; + } let name = event.name().to_ascii_lowercase(); if name.contains("generate_content") || name.contains("generatecontent") { OPERATION_GENERATE_CONTENT @@ -292,6 +384,16 @@ fn push_api_specific_request_attributes( attributes.push(KeyValue::new("gen_ai.request.seed", *value)); } } + Some(ApiSpecificRequest::Custom { api_name, data }) if api_name == EVALUATION_API_NAME => { + push_evaluation_identity_attributes(attributes, data); + if let Some(count) = data + .pointer("/value/questions") + .and_then(Value::as_object) + .and_then(|questions| i64::try_from(questions.len()).ok()) + { + attributes.push(KeyValue::new(EVALUATION_QUESTION_COUNT, count)); + } + } _ => {} } } @@ -339,6 +441,27 @@ fn push_llm_response_attributes(attributes: &mut Vec, event: &Event) { )); } } + if let Some(crate::codec::response::ApiSpecificResponse::Custom { api_name, data }) = + response.api_specific.as_ref() + && api_name == EVALUATION_API_NAME + { + push_evaluation_identity_attributes(attributes, data); + if let Some(answers) = data.pointer("/value/answers").and_then(Value::as_object) { + // Raw evaluation answers can contain provider-generated text and caller-chosen IDs. + // Keep that content in Relay's normal event pipeline, where redaction plugins can + // inspect it, and project only content-free aggregates into OTLP attributes. + push_evaluation_answer_counts(attributes, answers); + } + } +} + +fn push_evaluation_identity_attributes(attributes: &mut Vec, data: &Json) { + if let Some(provider) = data.get("provider").and_then(Value::as_str) { + attributes.push(KeyValue::new(EVALUATION_PROVIDER, provider.to_owned())); + } + if let Some(operation) = data.get("operation").and_then(Value::as_str) { + attributes.push(KeyValue::new(EVALUATION_OPERATION, operation.to_owned())); + } } fn gen_ai_input_tokens(event: &Event, response: &AnnotatedLlmResponse) -> Option { @@ -740,6 +863,7 @@ fn provider_from_event_name(event: &Event) -> Option { ("openai", "openai"), ("gpt", "openai"), ("perplexity", "perplexity"), + ("typesafe", "typesafe"), ] .into_iter() .find_map(|(needle, provider)| name.contains(needle).then(|| provider.to_string())) @@ -755,6 +879,9 @@ fn provider_from_normalized_request(event: &Event) -> Option<&'static str> { // Not an OTel well-known value yet; follows the dotted cloud-provider // convention (`aws.bedrock`, `gcp.gemini`). ApiSpecificRequest::OCIGenAI { .. } => Some("oci.genai"), + ApiSpecificRequest::Custom { api_name, .. } if api_name == EVALUATION_API_NAME => { + Some("typesafe") + } ApiSpecificRequest::Custom { .. } => None, } } diff --git a/crates/core/src/plugins/nemo_guardrails/python.rs b/crates/core/src/plugins/nemo_guardrails/python.rs index b50f6f490..392e09798 100644 --- a/crates/core/src/plugins/nemo_guardrails/python.rs +++ b/crates/core/src/plugins/nemo_guardrails/python.rs @@ -905,13 +905,14 @@ impl LocalGuardrailsCodec { } } - fn from_provider_surface(surface: ProviderSurface) -> Self { + fn from_provider_surface(surface: ProviderSurface) -> Option { match surface { - ProviderSurface::OpenAIChat => Self::OpenAIChat, - ProviderSurface::OpenAIResponses => Self::OpenAIResponses, - ProviderSurface::AnthropicMessages => Self::AnthropicMessages, - ProviderSurface::OCIGenAI => Self::OCIGenAI, - ProviderSurface::GeminiGenerateContent => Self::GeminiGenerateContent, + ProviderSurface::OpenAIChat => Some(Self::OpenAIChat), + ProviderSurface::OpenAIResponses => Some(Self::OpenAIResponses), + ProviderSurface::AnthropicMessages => Some(Self::AnthropicMessages), + ProviderSurface::OCIGenAI => Some(Self::OCIGenAI), + ProviderSurface::GeminiGenerateContent => Some(Self::GeminiGenerateContent), + ProviderSurface::TypeSafeSystemOne => None, } } @@ -942,7 +943,13 @@ fn resolve_codec(config: &NeMoGuardrailsConfig) -> PluginResult match ProviderSurface::from_codec_name(name) { - Some(surface) => Ok(Some(LocalGuardrailsCodec::from_provider_surface(surface))), + Some(surface) => LocalGuardrailsCodec::from_provider_surface(surface) + .map(Some) + .ok_or_else(|| { + PluginError::InvalidConfig(format!( + "local NeMo Guardrails does not support evaluation codec '{name}'" + )) + }), None => Err(PluginError::InvalidConfig(format!( "unsupported local NeMo Guardrails codec '{name}'" ))), diff --git a/crates/core/tests/unit/codec/resolve_tests.rs b/crates/core/tests/unit/codec/resolve_tests.rs index d8a60a3f3..10a6ad47e 100644 --- a/crates/core/tests/unit/codec/resolve_tests.rs +++ b/crates/core/tests/unit/codec/resolve_tests.rs @@ -23,6 +23,7 @@ fn builtin_provider_surface_registry_keeps_request_priority() { assert_eq!( surfaces, vec![ + ProviderSurface::TypeSafeSystemOne, ProviderSurface::OpenAIResponses, ProviderSurface::AnthropicMessages, ProviderSurface::OCIGenAI, @@ -448,7 +449,8 @@ fn hint_does_not_classify_non_object_or_keyless() { // Provider-codec factory (name<->surface mapping + codec construction) // --------------------------------------------------------------------------- -const ALL_SURFACES: [ProviderSurface; 5] = [ +const ALL_SURFACES: [ProviderSurface; 6] = [ + ProviderSurface::TypeSafeSystemOne, ProviderSurface::OpenAIChat, ProviderSurface::OpenAIResponses, ProviderSurface::AnthropicMessages, @@ -469,6 +471,10 @@ fn codec_name_round_trips_for_every_surface() { #[test] fn codec_name_uses_canonical_spellings() { + assert_eq!( + ProviderSurface::TypeSafeSystemOne.codec_name(), + "typesafe_system_one" + ); assert_eq!(ProviderSurface::OpenAIChat.codec_name(), "openai_chat"); assert_eq!( ProviderSurface::OpenAIResponses.codec_name(), @@ -511,6 +517,7 @@ fn supported_codec_names_track_the_builtin_registry() { assert_eq!( supported_codec_names(), vec![ + "typesafe_system_one", "openai_responses", "anthropic_messages", "oci_genai", @@ -654,9 +661,15 @@ fn streaming_codec_round_trips_through_its_response_codec() { } #[test] -fn streaming_codec_constructs_a_usable_codec_for_every_surface() { +fn streaming_codec_is_usable_or_explicitly_unsupported_for_every_surface() { for surface in ALL_SURFACES { - let assembled = streaming_codec(surface).finalizer()(); + let codec = streaming_codec(surface); + if surface == ProviderSurface::TypeSafeSystemOne { + assert!(codec.collector()(json!({})).is_err()); + assert!(codec.finalizer()().is_null()); + continue; + } + let assembled = codec.finalizer()(); assert!( assembled.is_object(), "{surface:?} streaming codec finalizes to a JSON object", diff --git a/crates/core/tests/unit/codec/typesafe_system_one_tests.rs b/crates/core/tests/unit/codec/typesafe_system_one_tests.rs new file mode 100644 index 000000000..37e632c1e --- /dev/null +++ b/crates/core/tests/unit/codec/typesafe_system_one_tests.rs @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::codec::response::{ + PricingCatalog, PricingResolver, reset_active_pricing_resolver, set_active_pricing_resolver, +}; +use serde_json::json; + +fn request(content: Json) -> LlmRequest { + LlmRequest { + headers: Map::new(), + content, + } +} + +fn fixture() -> Json { + json!({ + "model": "jev-latest", + "state": {"candidate": "The answer is 42."}, + "questions": { + "correct": { + "type": "noul", + "instructions": "Is the candidate correct?", + "criteria": { + "true": "correct", + "false": "incorrect", + "provider_label": "preserve-nested" + }, + "provider_extension": "preserve-me" + }, + "quality": { + "type": "choice", + "instructions": ["Choose a quality band"], + "criteria": {"high": {"meaning": "good"}, "low": null} + }, + "score": { + "type": "score", + "instructions": {"rubric": "Score the answer"}, + "criteria": ["wrong", {"meaning": "partly right"}, null] + } + }, + "request_id": "req-123" + }) +} + +#[test] +fn decodes_into_provider_neutral_evaluation_without_chat_semantics() { + let annotated = TypeSafeSystemOneCodec.decode(&request(fixture())).unwrap(); + + assert_eq!(annotated.model.as_deref(), Some("jev-latest")); + assert!(annotated.messages.is_empty()); + assert!(annotated.tools.is_none()); + assert_eq!(annotated.extra.get("request_id"), Some(&json!("req-123"))); + + let evaluation = evaluation_request(&annotated).unwrap(); + assert_eq!(evaluation.questions.len(), 3); + assert!(matches!( + evaluation.questions.get("correct"), + Some(EvaluationQuestion::Boolean { .. }) + )); + assert!(matches!( + evaluation.questions.get("quality"), + Some(EvaluationQuestion::Choice { .. }) + )); + assert!(matches!( + evaluation.questions.get("score"), + Some(EvaluationQuestion::Score { .. }) + )); +} + +#[test] +fn unchanged_round_trip_is_lossless() { + let original = request(fixture()); + let annotated = TypeSafeSystemOneCodec.decode(&original).unwrap(); + let encoded = TypeSafeSystemOneCodec + .encode(&annotated, &original) + .unwrap(); + assert_eq!(encoded, original); +} + +#[test] +fn edits_model_and_evaluation_while_preserving_unknown_fields() { + let original = request(fixture()); + let mut annotated = TypeSafeSystemOneCodec.decode(&original).unwrap(); + annotated.model = Some("jev-1.13.0".into()); + let ApiSpecificRequest::Custom { data, .. } = annotated.api_specific.as_mut().unwrap() else { + panic!("expected custom evaluation data"); + }; + let mut envelope: EvaluationEnvelope = + serde_json::from_value(data.clone()).unwrap(); + envelope.value.model = "jev-1.13.0".into(); + envelope.value.state = json!(["new", "state"]); + if let EvaluationQuestion::Boolean { instructions, .. } = envelope + .value + .questions + .get_mut("correct") + .expect("boolean question") + { + *instructions = json!("Updated instruction"); + } + *data = serde_json::to_value(envelope).unwrap(); + + let encoded = TypeSafeSystemOneCodec + .encode(&annotated, &original) + .unwrap(); + assert_eq!(encoded.content["model"], "jev-1.13.0"); + assert_eq!(encoded.content["state"], json!(["new", "state"])); + assert_eq!(encoded.content["request_id"], "req-123"); + assert_eq!( + encoded.content["questions"]["correct"]["provider_extension"], + "preserve-me" + ); + assert_eq!( + encoded.content["questions"]["correct"]["criteria"]["provider_label"], + "preserve-nested" + ); +} + +#[test] +fn choice_edits_can_remove_options_without_restoring_them_as_extensions() { + let original = request(fixture()); + let mut annotated = TypeSafeSystemOneCodec.decode(&original).unwrap(); + let ApiSpecificRequest::Custom { data, .. } = annotated.api_specific.as_mut().unwrap() else { + panic!("expected custom evaluation data"); + }; + let mut envelope: EvaluationEnvelope = + serde_json::from_value(data.clone()).unwrap(); + let EvaluationQuestion::Choice { criteria, .. } = envelope + .value + .questions + .get_mut("quality") + .expect("choice question") + else { + panic!("expected choice question"); + }; + criteria.remove("low"); + *data = serde_json::to_value(envelope).unwrap(); + + let encoded = TypeSafeSystemOneCodec + .encode(&annotated, &original) + .unwrap(); + assert_eq!( + encoded.content["questions"]["quality"]["criteria"], + json!({"high": {"meaning": "good"}}) + ); +} + +#[test] +fn rejects_modeled_fields_in_the_extra_map() { + let original = request(fixture()); + let mut annotated = TypeSafeSystemOneCodec.decode(&original).unwrap(); + annotated.extra.insert("model".into(), json!("shadowed")); + let error = TypeSafeSystemOneCodec + .encode(&annotated, &original) + .unwrap_err(); + assert!(error.to_string().contains("cannot be edited through extra")); +} + +#[test] +fn explicitly_rejects_every_streaming_declaration() { + for stream in [json!(true), json!(false), json!(null)] { + let mut body = fixture(); + body["stream"] = stream; + let error = TypeSafeSystemOneCodec.decode(&request(body)).unwrap_err(); + assert!(error.to_string().contains("does not support streaming")); + } +} + +#[test] +fn rejects_malformed_requests() { + for body in [ + json!({"model": "jev", "state": 7, "questions": {"q": {"type": "noul", "instructions": "x"}}}), + json!({"model": "jev", "state": "x", "questions": {}}), + json!({"model": "jev", "state": "x", "questions": {"q": {"type": "score", "instructions": "x", "criteria": []}}}), + ] { + assert!(TypeSafeSystemOneCodec.decode(&request(body)).is_err()); + } +} + +#[test] +fn accepts_python_sdk_single_score_and_choice_shapes() { + for question in [ + json!({"type": "score", "criteria": ["only"]}), + json!({"type": "choice", "criteria": {}}), + ] { + let body = json!({ + "model": "jev-latest", + "state": "candidate", + "questions": {"q": question} + }); + assert!(TypeSafeSystemOneCodec.decode(&request(body)).is_ok()); + } +} + +#[test] +fn accepts_null_state_and_omitted_optional_instructions() { + let body = json!({ + "model": "jev-latest", + "state": null, + "questions": {"q": {"type": "noul"}} + }); + let annotated = TypeSafeSystemOneCodec + .decode(&request(body.clone())) + .unwrap(); + assert_eq!(evaluation_request(&annotated).unwrap().state, Json::Null); + assert_eq!( + super::super::resolve::detect_request_surface(&body), + Some(ProviderSurface::TypeSafeSystemOne) + ); +} + +fn response_fixture(model: &str) -> Json { + json!({ + "model": model, + "answers": { + "correct": {"type": "noul", "noul": 0.93}, + "quality": { + "type": "choice", + "choice": "high", + "probabilities": {"high": 0.8, "low": 0.2}, + "confidence": 0.8 + }, + "score": { + "type": "score", + "score": 1.8, + "legend": {"0": "wrong", "1": "partial", "2": "right"}, + "probabilities": {"0": 0.05, "1": 0.1, "2": 0.85}, + "confidence": 0.85 + } + }, + "usage": {"billing_units": 42, "input_tokens": 1_000_000, "output_tokens": 0}, + "trace_id": "typesafe-trace" + }) +} + +#[test] +fn decodes_structured_answers_and_usage_without_text_output() { + let response = TypeSafeSystemOneCodec + .decode_response(&response_fixture("jev-1.13.0")) + .unwrap(); + + assert_eq!(response.model.as_deref(), Some("jev-1.13.0")); + assert!(response.message.is_none()); + assert!(response.finish_reason.is_none()); + let usage = response.usage.as_ref().unwrap(); + assert_eq!(usage.prompt_tokens, Some(1_000_000)); + assert_eq!(usage.completion_tokens, Some(0)); + assert_eq!(usage.total_tokens, Some(1_000_000)); + assert_eq!( + response.extra.get("trace_id"), + Some(&json!("typesafe-trace")) + ); + + let evaluation = evaluation_response(&response).unwrap(); + assert_eq!(evaluation.answers.len(), 3); + assert_eq!(evaluation.usage.unwrap().billing_units, Some(42)); +} + +#[test] +fn rejects_invalid_answer_probabilities() { + let mut response = response_fixture("jev-1.13.0"); + response["answers"]["correct"]["noul"] = json!(1.01); + assert!(TypeSafeSystemOneCodec.decode_response(&response).is_err()); +} + +#[test] +fn uses_provider_scoped_model_alias_pricing() { + struct ResetPricing; + impl Drop for ResetPricing { + fn drop(&mut self) { + let _ = reset_active_pricing_resolver(); + } + } + let _reset = ResetPricing; + let catalog = PricingCatalog::from_json_str( + &json!({ + "version": 1, + "entries": [{ + "provider": "typesafe", + "model_id": "jev-1.13.0", + "aliases": ["jev-latest"], + "pricing_as_of": "2026-09-17", + "pricing_source": "https://www.typesafe.ai/", + "rates": {"input_per_million": 0.042, "output_per_million": 0.0}, + "prompt_cache": {"read_accounting": "included_in_prompt_tokens"} + }] + }) + .to_string(), + ) + .unwrap(); + set_active_pricing_resolver(PricingResolver::from_catalogs(vec![catalog])).unwrap(); + + let response = TypeSafeSystemOneCodec + .decode_response(&response_fixture("jev-latest")) + .unwrap(); + let cost = response.usage.unwrap().cost.unwrap(); + assert_eq!(cost.total, Some(0.042)); + assert_eq!(cost.pricing_provider.as_deref(), Some("typesafe")); + assert_eq!(cost.pricing_model.as_deref(), Some("jev-1.13.0")); +} + +#[test] +fn exposes_builtin_identity_and_surface_detection() { + assert_eq!( + LlmCodec::codec_identity(&TypeSafeSystemOneCodec), + LlmCodecIdentity::BuiltIn(BuiltinLlmCodec::TypeSafeSystemOne) + ); + assert_eq!( + super::super::resolve::detect_request_surface(&fixture()), + Some(ProviderSurface::TypeSafeSystemOne) + ); + assert_eq!( + super::super::resolve::detect_response_surface(&response_fixture("jev-latest")), + Some(ProviderSurface::TypeSafeSystemOne) + ); +} diff --git a/crates/core/tests/unit/evaluation_api_tests.rs b/crates/core/tests/unit/evaluation_api_tests.rs new file mode 100644 index 000000000..5fc423400 --- /dev/null +++ b/crates/core/tests/unit/evaluation_api_tests.rs @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![allow(clippy::await_holding_lock)] // Serializes access to process-wide runtime state. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use serde_json::json; + +use super::{EvaluationExecuteParams, EvaluationExecutionNextFn, evaluation_execute}; +use crate::api::event::{Event, ScopeCategory}; +use crate::api::runtime::{NemoRelayContextState, global_context}; +use crate::api::scope::ScopeType; +use crate::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; +use crate::evaluation::{ + EvaluationAnswer, EvaluationQuestion, EvaluationRequest, EvaluationResponse, EvaluationUsage, +}; + +fn reset_global() { + crate::shared_runtime::reset_runtime_owner_for_tests(); + *global_context().write().unwrap() = NemoRelayContextState::new(); +} + +#[tokio::test] +async fn managed_evaluation_emits_evaluator_input_and_output() { + let _guard = crate::shared_runtime::runtime_owner_test_mutex() + .lock() + .unwrap_or_else(|error| error.into_inner()); + reset_global(); + let captured = Arc::new(Mutex::new(Vec::::new())); + let subscriber_events = captured.clone(); + register_subscriber( + "evaluation-api-observer", + Arc::new(move |event| subscriber_events.lock().unwrap().push(event.clone())), + ) + .unwrap(); + + let request = EvaluationRequest { + model: "jev-latest".into(), + state: json!({"candidate": "42"}), + questions: BTreeMap::from([( + "correct".into(), + EvaluationQuestion::Boolean { + instructions: json!("Is this correct?"), + criteria: None, + }, + )]), + }; + let callback: EvaluationExecutionNextFn = Arc::new(|request| { + Box::pin(async move { + Ok(EvaluationResponse { + model: request.model, + answers: BTreeMap::from([( + "correct".into(), + EvaluationAnswer::Boolean { probability: 0.9 }, + )]), + usage: Some(EvaluationUsage { + billing_units: None, + input_tokens: Some(7), + output_tokens: Some(0), + }), + }) + }) + }); + let response = evaluation_execute( + EvaluationExecuteParams::builder() + .name("typesafe.system_one") + .request(request) + .func(callback) + .build(), + ) + .await + .unwrap(); + assert_eq!(response.model, "jev-latest"); + + flush_subscribers().unwrap(); + assert!(deregister_subscriber("evaluation-api-observer").unwrap()); + let events = captured.lock().unwrap(); + let start = events + .iter() + .find(|event| event.scope_category() == Some(ScopeCategory::Start)) + .unwrap(); + let end = events + .iter() + .find(|event| event.scope_category() == Some(ScopeCategory::End)) + .unwrap(); + assert_eq!(start.scope_type(), Some(ScopeType::Evaluator)); + assert_eq!(start.input().unwrap()["model"], "jev-latest"); + assert_eq!( + end.output().unwrap()["answers"]["correct"]["probability"], + 0.9 + ); + assert_eq!(end.metadata().unwrap()["otel.status_code"], "OK"); +} diff --git a/crates/core/tests/unit/observability/otel_tests.rs b/crates/core/tests/unit/observability/otel_tests.rs index d98eac4fd..8e558f3f9 100644 --- a/crates/core/tests/unit/observability/otel_tests.rs +++ b/crates/core/tests/unit/observability/otel_tests.rs @@ -2409,6 +2409,12 @@ fn gen_ai_projection_uses_standard_operation_names_and_span_kinds() { "retrieval", SpanKind::Client, ), + ( + ScopeType::Evaluator, + "typesafe.system_one", + "evaluate", + SpanKind::Client, + ), ] { let event = make_start_event(Uuid::now_v7(), None, name, scope_type, None); assert_eq!( @@ -2425,7 +2431,6 @@ fn gen_ai_projection_uses_standard_operation_names_and_span_kinds() { ScopeType::Function, ScopeType::Reranker, ScopeType::Guardrail, - ScopeType::Evaluator, ScopeType::Custom, ScopeType::Unknown, ] { @@ -2442,6 +2447,76 @@ fn gen_ai_projection_uses_standard_operation_names_and_span_kinds() { } } +#[test] +fn gen_ai_projection_covers_provider_neutral_evaluator_scopes() { + let uuid = Uuid::now_v7(); + let start = make_start_event( + uuid, + None, + "typesafe.system_one", + ScopeType::Evaluator, + Some(json!({ + "model": "jev-latest", + "state": {"candidate": "sensitive"}, + "questions": { + "correct": {"type": "boolean", "instructions": "sensitive"}, + "quality": {"type": "choice", "instructions": "sensitive", "criteria": {}} + } + })), + ); + assert_eq!( + crate::observability::otel_genai::span_name(&start), + "evaluate jev-latest" + ); + let start = attr_map(&crate::observability::otel_genai::start_attributes(&start)); + assert_eq!( + start.get("gen_ai.operation.name"), + Some(&"evaluate".to_string()) + ); + assert_eq!( + start.get("gen_ai.provider.name"), + Some(&"typesafe".to_string()) + ); + assert_eq!( + start.get("gen_ai.request.model"), + Some(&"jev-latest".to_string()) + ); + assert_eq!( + start.get("nemo_relay.evaluation.question_count"), + Some(&"2".to_string()) + ); + assert!(start.values().all(|value| !value.contains("sensitive"))); + + let end = make_end_event( + uuid, + None, + "typesafe.system_one", + ScopeType::Evaluator, + Some(json!({ + "model": "jev-1.13.0", + "answers": { + "correct": {"type": "boolean", "probability": 0.9}, + "quality": {"type": "choice", "choice": "sensitive", "probabilities": {}} + }, + "usage": {"billing_units": 1, "input_tokens": 12, "output_tokens": 0} + })), + ); + let end = attr_map(&crate::observability::otel_genai::end_attributes(&end)); + for (key, expected) in [ + ("gen_ai.response.model", "jev-1.13.0"), + ("gen_ai.usage.input_tokens", "12"), + ("gen_ai.usage.output_tokens", "0"), + ("nemo_relay.evaluation.billing_units", "1"), + ("nemo_relay.evaluation.answer_count", "2"), + ("nemo_relay.evaluation.boolean_answer_count", "1"), + ("nemo_relay.evaluation.choice_answer_count", "1"), + ("nemo_relay.evaluation.score_answer_count", "0"), + ] { + assert_eq!(end.get(key), Some(&expected.to_string()), "{key}"); + } + assert!(end.values().all(|value| !value.contains("sensitive"))); +} + #[test] fn gen_ai_projection_emits_only_span_specific_attributes() { let common = json!({ @@ -3593,6 +3668,125 @@ fn gen_ai_projection_covers_optional_request_controls_and_finish_reasons() { } } +#[test] +fn gen_ai_projection_exposes_system_one_as_structured_evaluation() { + let request = make_scope_event_with_profile( + ScopeCategory::Start, + Uuid::now_v7(), + None, + "typesafe.system_one", + ScopeType::Llm, + None, + Some( + CategoryProfile::builder() + .annotated_request(Arc::new(AnnotatedLlmRequest { + model: Some("jev-latest".to_string()), + api_specific: Some(crate::codec::request::ApiSpecificRequest::Custom { + api_name: "typesafe.system_one".to_string(), + data: json!({ + "provider": "typesafe", + "operation": "system_one", + "value": { + "model": "jev-latest", + "state": {"candidate": "42"}, + "questions": { + "correct": { + "type": "boolean", + "instructions": "Is this correct?" + }, + "quality": { + "type": "score", + "instructions": "Score it", + "criteria": ["bad", "good"] + } + } + } + }), + }), + ..AnnotatedLlmRequest::default() + })) + .build(), + ), + ); + let request_attributes = attr_map(&crate::observability::otel_genai::start_attributes( + &request, + )); + assert_eq!( + crate::observability::otel_genai::span_name(&request), + "evaluate jev-latest" + ); + for (key, expected) in [ + ("gen_ai.operation.name", "evaluate"), + ("gen_ai.provider.name", "typesafe"), + ("nemo_relay.evaluation.provider", "typesafe"), + ("nemo_relay.evaluation.operation", "system_one"), + ("nemo_relay.evaluation.question_count", "2"), + ] { + assert_eq!(request_attributes.get(key), Some(&expected.to_string())); + } + assert!(!request_attributes.contains_key("gen_ai.input.messages")); + + let response = make_scope_event_with_profile( + ScopeCategory::End, + Uuid::now_v7(), + None, + "typesafe.system_one", + ScopeType::Llm, + None, + Some( + CategoryProfile::builder() + .annotated_response(Arc::new(AnnotatedLlmResponse { + model: Some("jev-1.13.0".to_string()), + usage: Some(Usage { + prompt_tokens: Some(12), + completion_tokens: Some(0), + total_tokens: Some(12), + ..Usage::default() + }), + api_specific: Some(crate::codec::response::ApiSpecificResponse::Custom { + api_name: "typesafe.system_one".to_string(), + data: json!({ + "provider": "typesafe", + "operation": "system_one", + "value": { + "model": "jev-1.13.0", + "answers": { + "correct": {"type": "boolean", "probability": 0.9}, + "quality": { + "type": "score", + "score": 1.8, + "legend": {"0": "bad", "1": "good"}, + "probabilities": {"0": 0.1, "1": 0.9} + } + }, + "usage": {"billing_units": 1, "input_tokens": 12} + } + }), + }), + ..empty_annotated_response() + })) + .build(), + ), + ); + let response_attributes = + attr_map(&crate::observability::otel_genai::end_attributes(&response)); + for (key, expected) in [ + ("gen_ai.response.model", "jev-1.13.0"), + ("gen_ai.usage.input_tokens", "12"), + ("gen_ai.usage.output_tokens", "0"), + ("nemo_relay.evaluation.provider", "typesafe"), + ("nemo_relay.evaluation.operation", "system_one"), + ("nemo_relay.evaluation.answer_count", "2"), + ("nemo_relay.evaluation.boolean_answer_count", "1"), + ("nemo_relay.evaluation.choice_answer_count", "0"), + ("nemo_relay.evaluation.score_answer_count", "1"), + ] { + assert_eq!(response_attributes.get(key), Some(&expected.to_string())); + } + assert!(!response_attributes.contains_key("nemo_relay.evaluation.answers")); + assert!(!response_attributes.contains_key("gen_ai.output.messages")); +} + #[test] fn gen_ai_projection_covers_message_variants_and_empty_input() { let annotated_request = serde_json::from_value::(json!({ diff --git a/crates/core/tests/unit/plugins/nemo_guardrails/local_python_tests.rs b/crates/core/tests/unit/plugins/nemo_guardrails/local_python_tests.rs index 10dde622d..7003fbd87 100644 --- a/crates/core/tests/unit/plugins/nemo_guardrails/local_python_tests.rs +++ b/crates/core/tests/unit/plugins/nemo_guardrails/local_python_tests.rs @@ -696,7 +696,9 @@ fn local_codec_and_rewrite_helpers_cover_all_provider_surfaces() { ] { assert_eq!(codec.provider_surface(), surface); assert_eq!( - LocalGuardrailsCodec::from_provider_surface(surface).provider_surface(), + LocalGuardrailsCodec::from_provider_surface(surface) + .expect("chat provider surface") + .provider_surface(), surface ); } diff --git a/crates/ffi/nemo_relay.h b/crates/ffi/nemo_relay.h index ba95f9465..d0b5d8da0 100644 --- a/crates/ffi/nemo_relay.h +++ b/crates/ffi/nemo_relay.h @@ -1217,6 +1217,18 @@ struct FfiCodecHandle *nemo_relay_anthropic_messages_codec_new(void); */ struct FfiCodecHandle *nemo_relay_gemini_generate_content_codec_new(void); +/** + * Create a new TypeSafe System One evaluation codec handle. + * + * The returned handle implements request decode/encode and response decode. + * System One requests are explicitly non-streaming. Free with + * `nemo_relay_codec_free`. + * + * # Safety + * Caller must free the returned handle via `nemo_relay_codec_free`. + */ +struct FfiCodecHandle *nemo_relay_typesafe_system_one_codec_new(void); + /** * Execute an LLM call end-to-end: run conditional-execution guardrails (on raw * request), then request intercepts, sanitize-request guardrails, execution diff --git a/crates/ffi/src/api/llm.rs b/crates/ffi/src/api/llm.rs index 31ba4f8e9..23e1b9a3f 100644 --- a/crates/ffi/src/api/llm.rs +++ b/crates/ffi/src/api/llm.rs @@ -389,6 +389,22 @@ pub extern "C" fn nemo_relay_gemini_generate_content_codec_new() -> *mut FfiCode })) } +/// Create a new TypeSafe System One evaluation codec handle. +/// +/// The returned handle implements request decode/encode and response decode. +/// System One requests are explicitly non-streaming. Free with +/// `nemo_relay_codec_free`. +/// +/// # Safety +/// Caller must free the returned handle via `nemo_relay_codec_free`. +#[unsafe(no_mangle)] +pub extern "C" fn nemo_relay_typesafe_system_one_codec_new() -> *mut FfiCodecHandle { + Box::into_raw(Box::new(FfiCodecHandle { + codec: Arc::new(nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec), + response_codec: Arc::new(nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec), + })) +} + struct ParsedExecuteInputs { name: String, request: LlmRequest, diff --git a/crates/ffi/tests/unit/types_tests.rs b/crates/ffi/tests/unit/types_tests.rs index 675715693..f63a12d58 100644 --- a/crates/ffi/tests/unit/types_tests.rs +++ b/crates/ffi/tests/unit/types_tests.rs @@ -762,6 +762,7 @@ fn test_annotated_event_accessors_and_codec_handles() { let openai_responses = api::nemo_relay_openai_responses_codec_new(); let anthropic = api::nemo_relay_anthropic_messages_codec_new(); let gemini = api::nemo_relay_gemini_generate_content_codec_new(); + let typesafe = api::nemo_relay_typesafe_system_one_codec_new(); assert!(!openai_chat.is_null()); assert!(!openai_responses.is_null()); assert!(!anthropic.is_null()); @@ -769,12 +770,14 @@ fn test_annotated_event_accessors_and_codec_handles() { !gemini.is_null(), "GeminiGenerateContentCodec FFI constructor must return a non-null handle" ); + assert!(!typesafe.is_null()); unsafe { nemo_relay_codec_free(openai_chat); nemo_relay_codec_free(openai_responses); nemo_relay_codec_free(anthropic); nemo_relay_codec_free(gemini); + nemo_relay_codec_free(typesafe); nemo_relay_codec_free(std::ptr::null_mut()); } } diff --git a/crates/node/src/types/mod.rs b/crates/node/src/types/mod.rs index fb445ef0d..71fb4c34f 100644 --- a/crates/node/src/types/mod.rs +++ b/crates/node/src/types/mod.rs @@ -726,6 +726,64 @@ impl GeminiGenerateContentCodec { } } +/// Built-in codec for the non-streaming TypeSafe System One evaluation API. +#[napi(js_name = "TypeSafeSystemOneCodec")] +pub struct TypeSafeSystemOneCodec { + pub(crate) inner_codec: std::sync::Arc, + pub(crate) inner_response_codec: std::sync::Arc, +} + +#[napi] +impl TypeSafeSystemOneCodec { + #[napi(constructor)] + pub fn new() -> Self { + Self { + inner_codec: std::sync::Arc::new( + nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec, + ), + inner_response_codec: std::sync::Arc::new( + nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec, + ), + } + } + + /// Decode an opaque System One request into normalized evaluation data. + #[napi] + pub fn decode(&self, request: Json) -> napi::Result { + let llm_req: CoreLlmRequest = serde_json::from_value(request) + .map_err(|e| napi::Error::from_reason(format!("invalid LlmRequest: {e}")))?; + let annotated = self + .inner_codec + .decode(&llm_req) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + serde_json::to_value(&annotated).map_err(|e| napi::Error::from_reason(e.to_string())) + } + + /// Encode normalized evaluation changes back into a System One request. + #[napi] + pub fn encode(&self, annotated: Json, original: Json) -> napi::Result { + let ann: AnnotatedLlmRequest = serde_json::from_value(annotated) + .map_err(|e| napi::Error::from_reason(format!("invalid AnnotatedLlmRequest: {e}")))?; + let orig: CoreLlmRequest = serde_json::from_value(original) + .map_err(|e| napi::Error::from_reason(format!("invalid LlmRequest: {e}")))?; + let result = self + .inner_codec + .encode(&ann, &orig) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + serde_json::to_value(&result).map_err(|e| napi::Error::from_reason(e.to_string())) + } + + /// Decode a System One response into normalized evaluation data and usage. + #[napi(js_name = "decodeResponse")] + pub fn decode_response(&self, response: Json) -> napi::Result { + let annotated = self + .inner_response_codec + .decode_response(&response) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + serde_json::to_value(&annotated).map_err(|e| napi::Error::from_reason(e.to_string())) + } +} + /// Built-in codec for the Anthropic Messages API. /// /// Implements both request codec (decode/encode) and response codec diff --git a/crates/node/tests/types_tests.mjs b/crates/node/tests/types_tests.mjs index 55692319d..71ade992b 100644 --- a/crates/node/tests/types_tests.mjs +++ b/crates/node/tests/types_tests.mjs @@ -27,6 +27,7 @@ describe('Type constants', () => { assert.equal(typeof lib.AnthropicMessagesCodec, 'function'); assert.equal(typeof lib.OCIGenAIChatCodec, 'function'); assert.equal(typeof lib.GeminiGenerateContentCodec, 'function'); + assert.equal(typeof lib.TypeSafeSystemOneCodec, 'function'); }); it('scope type enum values', () => { @@ -69,6 +70,48 @@ describe('Type constants', () => { }); }); +describe('TypeSafeSystemOneCodec', () => { + const { TypeSafeSystemOneCodec } = lib; + + it('round-trips non-streaming evaluations and decodes decisions', () => { + const codec = new TypeSafeSystemOneCodec(); + const original = { + headers: {}, + content: { + model: 'jev-latest', + state: { candidate: '42' }, + questions: { correct: { type: 'noul', instructions: 'Correct?' } }, + request_id: 'preserved', + }, + }; + const annotated = codec.decode(original); + assert.deepEqual(codec.encode(annotated, original), original); + const response = codec.decodeResponse({ + model: 'jev-1.13.0', + answers: { correct: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 12, output_tokens: 0 }, + }); + assert.equal(response.model, 'jev-1.13.0'); + assert.equal(response.usage.prompt_tokens, 12); + }); + + it('rejects explicit streaming', () => { + const codec = new TypeSafeSystemOneCodec(); + assert.throws( + () => codec.decode({ + headers: {}, + content: { + model: 'jev-latest', + state: 'candidate', + questions: { correct: { type: 'noul', instructions: 'Correct?' } }, + stream: false, + }, + }), + /does not support streaming/, + ); + }); +}); + // =========================================================================== // LlmRequest // =========================================================================== diff --git a/crates/pii-redaction/src/builtin.rs b/crates/pii-redaction/src/builtin.rs index ca733fe9c..3e7f9903f 100644 --- a/crates/pii-redaction/src/builtin.rs +++ b/crates/pii-redaction/src/builtin.rs @@ -615,6 +615,9 @@ impl CompiledBuiltinBackend { LlmCodecIdentity::BuiltIn(BuiltinLlmCodec::GeminiGenerateContent) => { Some(ProviderSurface::GeminiGenerateContent) } + // Evaluation payloads require a dedicated state/question redaction + // contract; do not reinterpret them as chat trajectories. + LlmCodecIdentity::BuiltIn(BuiltinLlmCodec::TypeSafeSystemOne) => None, LlmCodecIdentity::Runtime(_) | LlmCodecIdentity::Opaque => None, } } diff --git a/crates/pii-redaction/src/overlay.rs b/crates/pii-redaction/src/overlay.rs index fcfb26e8d..13a0546df 100644 --- a/crates/pii-redaction/src/overlay.rs +++ b/crates/pii-redaction/src/overlay.rs @@ -14,6 +14,7 @@ pub(crate) enum BuiltinCodecName { AnthropicMessages, OCIGenAI, GeminiGenerateContent, + TypeSafeSystemOne, } impl BuiltinCodecName { @@ -24,6 +25,7 @@ impl BuiltinCodecName { ProviderSurface::AnthropicMessages => Self::AnthropicMessages, ProviderSurface::OCIGenAI => Self::OCIGenAI, ProviderSurface::GeminiGenerateContent => Self::GeminiGenerateContent, + ProviderSurface::TypeSafeSystemOne => Self::TypeSafeSystemOne, } } @@ -38,6 +40,8 @@ impl BuiltinCodecName { Self::AnthropicMessages => overlay_anthropic_response(payload, annotated), Self::OCIGenAI => overlay_oci_genai_response(payload, annotated), Self::GeminiGenerateContent => overlay_gemini_response(payload, annotated), + // Evaluation answers have no chat response overlay contract. + Self::TypeSafeSystemOne => payload, } } } diff --git a/crates/pii-redaction/src/trajectory.rs b/crates/pii-redaction/src/trajectory.rs index 4fbf92eff..93513bbef 100644 --- a/crates/pii-redaction/src/trajectory.rs +++ b/crates/pii-redaction/src/trajectory.rs @@ -1198,7 +1198,8 @@ fn provider_hint_for_surface(surface: ProviderSurface) -> Option<&'static str> { ProviderSurface::OCIGenAI => Some("oci.genai"), ProviderSurface::OpenAIChat | ProviderSurface::OpenAIResponses - | ProviderSurface::GeminiGenerateContent => None, + | ProviderSurface::GeminiGenerateContent + | ProviderSurface::TypeSafeSystemOne => None, } } diff --git a/crates/pii-redaction/src/trajectory_projection.rs b/crates/pii-redaction/src/trajectory_projection.rs index 1139314fd..dc0ac0421 100644 --- a/crates/pii-redaction/src/trajectory_projection.rs +++ b/crates/pii-redaction/src/trajectory_projection.rs @@ -38,7 +38,8 @@ pub(super) fn render_request( ProviderSurface::OCIGenAI => Some("oci.genai"), ProviderSurface::OpenAIChat | ProviderSurface::OpenAIResponses - | ProviderSurface::GeminiGenerateContent => None, + | ProviderSurface::GeminiGenerateContent + | ProviderSurface::TypeSafeSystemOne => None, }; (detect_request_surface_with_hint(&rendered.content, provider_hint) == Some(surface)) .then_some(())?; @@ -61,6 +62,7 @@ pub(super) fn render_response( ProviderSurface::AnthropicMessages => render_anthropic_response(response), ProviderSurface::OCIGenAI => render_oci_response(response), ProviderSurface::GeminiGenerateContent => render_gemini_response(response), + ProviderSurface::TypeSafeSystemOne => return None, }; (detect_response_surface(&rendered) == Some(surface)).then_some(())?; response_codec(surface).decode_response(&rendered).ok()?; @@ -75,6 +77,7 @@ fn request_template(surface: ProviderSurface, request: &mut AnnotatedLlmRequest) ProviderSurface::OpenAIResponses => json!({"input": []}), ProviderSurface::OCIGenAI => oci_request_template(request)?, ProviderSurface::GeminiGenerateContent => json!({"contents": []}), + ProviderSurface::TypeSafeSystemOne => return None, }) } diff --git a/crates/python/src/py_types/codecs.rs b/crates/python/src/py_types/codecs.rs index 2b18c1087..31fe5d3a3 100644 --- a/crates/python/src/py_types/codecs.rs +++ b/crates/python/src/py_types/codecs.rs @@ -1149,3 +1149,62 @@ impl PyGeminiGenerateContentCodec { "" } } + +/// Built-in codec for TypeSafe System One evaluation requests and responses. +/// +/// System One is non-streaming and is represented as provider-neutral +/// evaluation data rather than chat messages. +#[pyclass(name = "TypeSafeSystemOneCodec")] +pub struct PyTypeSafeSystemOneCodec { + pub(crate) inner_codec: Arc, + pub(crate) inner_response_codec: Arc, +} + +#[pymethods] +impl PyTypeSafeSystemOneCodec { + #[new] + pub(crate) fn new() -> Self { + Self { + inner_codec: Arc::new(nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec), + inner_response_codec: Arc::new( + nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec, + ), + } + } + + /// Parse an opaque ``LLMRequest`` into a normalized evaluation request. + pub(crate) fn decode(&self, request: &PyLLMRequest) -> PyResult { + self.inner_codec + .decode(&request.inner) + .map(|r| PyAnnotatedLLMRequest { inner: r }) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Merge normalized evaluation changes back into the provider request. + pub(crate) fn encode( + &self, + annotated: &PyAnnotatedLLMRequest, + original: &PyLLMRequest, + ) -> PyResult { + self.inner_codec + .encode(&annotated.inner, &original.inner) + .map(|r| PyLLMRequest { inner: r }) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Parse a System One response into normalized evaluation data and usage. + pub(crate) fn decode_response( + &self, + response: &Bound<'_, PyAny>, + ) -> PyResult { + let json = py_to_json(response)?; + self.inner_response_codec + .decode_response(&json) + .map(|r| PyAnnotatedLLMResponse { inner: r }) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + pub(crate) fn __repr__(&self) -> &'static str { + "" + } +} diff --git a/crates/python/src/py_types/mod.rs b/crates/python/src/py_types/mod.rs index 1da1b1cc4..1ebcaddad 100644 --- a/crates/python/src/py_types/mod.rs +++ b/crates/python/src/py_types/mod.rs @@ -201,6 +201,7 @@ fn register_codec_types(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/crates/types/src/codec/identity.rs b/crates/types/src/codec/identity.rs index d888477ad..c1207f5ce 100644 --- a/crates/types/src/codec/identity.rs +++ b/crates/types/src/codec/identity.rs @@ -55,6 +55,8 @@ builtin_llm_codecs! { OCIGenAI => "oci_genai", /// Gemini generateContent request and response payloads. GeminiGenerateContent => "gemini_generate_content", + /// TypeSafe System One evaluation request and response payloads. + TypeSafeSystemOne => "typesafe_system_one", } /// Per-call LLM codec identity supplied to sanitizer and SDK callbacks. diff --git a/crates/types/src/evaluation.rs b/crates/types/src/evaluation.rs new file mode 100644 index 000000000..6661cd5d5 --- /dev/null +++ b/crates/types/src/evaluation.rs @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Provider-neutral decision and evaluation data types. +//! +//! Evaluation operations inspect one state value and answer a declared set of +//! typed questions. They are intentionally separate from chat messages and +//! generated text so providers such as TypeSafe System One can be represented +//! without inventing conversational semantics. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::Json; + +/// One provider-neutral evaluation request. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EvaluationRequest { + /// Model identifier requested by the caller. + pub model: String, + /// String, object, array, or null describing the state to evaluate. + pub state: Json, + /// Named questions evaluated over the shared state. + pub questions: BTreeMap, +} + +/// A typed question whose answer space is declared before inference. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EvaluationQuestion { + /// A binary judgment represented as the probability that the proposition is true. + Boolean { + /// Question or rubric supplied to the evaluator. + instructions: Json, + /// Optional descriptions of the true and false outcomes. + #[serde(skip_serializing_if = "Option::is_none")] + criteria: Option, + }, + /// Selection from a caller-declared set of options. + Choice { + /// Question or rubric supplied to the evaluator. + instructions: Json, + /// Option names and optional descriptions. + criteria: BTreeMap, + }, + /// Rating against an ordered caller-declared rubric. + Score { + /// Question or rubric supplied to the evaluator. + instructions: Json, + /// Ordered descriptions from the lowest to highest score. + criteria: Vec, + }, +} + +/// Optional semantic labels for a binary evaluation question. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BooleanCriteria { + /// Meaning of a result approaching one. + #[serde(rename = "true", skip_serializing_if = "Option::is_none")] + pub true_description: Option, + /// Meaning of a result approaching zero. + #[serde(rename = "false", skip_serializing_if = "Option::is_none")] + pub false_description: Option, +} + +/// One provider-neutral evaluation response. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EvaluationResponse { + /// Model version that performed the evaluation. + pub model: String, + /// Answers keyed by the question IDs from the request. + pub answers: BTreeMap, + /// Provider token usage, when reported. + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +/// A typed answer produced by an evaluation model. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EvaluationAnswer { + /// Probability that a binary proposition is true. + Boolean { + /// Probability in the inclusive range zero to one. + probability: f64, + }, + /// Selected option plus the complete option distribution. + Choice { + /// Highest-probability option. + choice: String, + /// Probability for every declared option. + probabilities: BTreeMap, + /// Provider-calculated confidence, when available. + #[serde(skip_serializing_if = "Option::is_none")] + confidence: Option, + }, + /// Probability-weighted position on an ordered rubric. + Score { + /// Weighted score; values may fall between rubric indices. + score: f64, + /// Rubric index to description mapping. + legend: BTreeMap, + /// Probability for every rubric index. + probabilities: BTreeMap, + /// Provider-calculated confidence, when available. + #[serde(skip_serializing_if = "Option::is_none")] + confidence: Option, + }, +} + +/// Token accounting returned by an evaluation provider. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct EvaluationUsage { + /// Provider billing units charged for this request, when reported. + #[serde(skip_serializing_if = "Option::is_none")] + pub billing_units: Option, + /// Tokens consumed by the input state, questions, and rubrics. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + /// Tokens or equivalent units consumed by answer production. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, +} diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index f36793fb7..11dba3548 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -13,6 +13,8 @@ pub mod api; /// Normalized LLM request and response data types. pub mod codec; +/// Provider-neutral decision and evaluation data types. +pub mod evaluation; /// Plugin configuration diagnostic data types. pub mod plugin; diff --git a/docs/daemon/configuration.mdx b/docs/daemon/configuration.mdx index 1ab8e6aaa..a7781b28c 100644 --- a/docs/daemon/configuration.mdx +++ b/docs/daemon/configuration.mdx @@ -41,6 +41,7 @@ For a new deployment, create `config.toml` with these contents: [upstream] openai_base_url = "https://api.openai.com/v1" anthropic_base_url = "https://api.anthropic.com" +typesafe_base_url = "https://api.typesafe.ai/v1" [logging] level = "info" diff --git a/docs/integrate-into-frameworks/provider-codecs.mdx b/docs/integrate-into-frameworks/provider-codecs.mdx index 88f24c646..fb2320880 100644 --- a/docs/integrate-into-frameworks/provider-codecs.mdx +++ b/docs/integrate-into-frameworks/provider-codecs.mdx @@ -94,6 +94,9 @@ Use the built-in provider codecs when the framework payload already matches a su - `OCIGenAIChatCodec`: OCI Generative AI chat-compatible requests and responses in the `GENERIC`, `COHERE`, and `COHEREV2` API formats. - `GeminiGenerateContentCodec`: Gemini `generateContent`-compatible requests and responses. +- `TypeSafeSystemOneCodec`: TypeSafe System One evaluation requests and + responses. This codec exposes typed evaluation data through `api_specific` + and intentionally does not synthesize chat messages or generated text. ## Provider Codec Roles diff --git a/docs/integrate-into-frameworks/typesafe-system-one.mdx b/docs/integrate-into-frameworks/typesafe-system-one.mdx new file mode 100644 index 000000000..0bea97377 --- /dev/null +++ b/docs/integrate-into-frameworks/typesafe-system-one.mdx @@ -0,0 +1,213 @@ +--- +title: "TypeSafe System One and Jev" +description: "First-class TypeSafe System One evaluation support through Relay." +position: 9 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + + +This integration treats Jev as a typed evaluation model, not as a chat model. +It supports TypeSafe's `noul`, `choice`, and `score` questions and preserves +their probabilities, confidence, model version, usage, top-level provider +extensions, and request-question extensions. + +TypeSafe describes System One as structured state in and typed probabilistic +decisions out. The public SDK contract and current API are early access, so pin +concrete model versions for repeatable workflows and keep conformance fixtures +current with the [official Python SDK](https://github.com/typesafe-ai/typesafe-sdk-python) +or [official JavaScript SDK](https://github.com/typesafe-ai/typesafe-sdk-js). + +## Gateway routes and official SDKs + +Relay accepts both `POST /systemone` and `POST /v1/systemone`. The default +upstream is `https://api.typesafe.ai/v1`, producing the canonical upstream URL +`https://api.typesafe.ai/v1/systemone`. + +For the official TypeSafe Python and JavaScript SDKs, use Relay's namespaced +base URL. The namespace keeps TypeSafe's `GET /v1/models` separate from the +OpenAI model catalog: + +```python +from typesafe_sdk import TypeSafeClient + +client = TypeSafeClient( + api_key="relay-client-credential-or-typesafe-key", + base_url="http://127.0.0.1:4040/typesafe", +) +models = client.models.list() +``` + +```ts +import { TypeSafeClient } from "@typesafe-ai/sdk"; + +const client = new TypeSafeClient({ + apiKey: "relay-client-credential-or-typesafe-key", + baseURL: "http://127.0.0.1:4040/typesafe", +}); +const models = await client.models.list(); +``` + +Relay maps `POST /typesafe/v1/systemone` and `GET /typesafe/v1/models` back to +the provider's canonical `/v1` paths. The unprefixed System One routes remain +available for curl and existing integrations. + +```bash +export TYPESAFE_API_KEY="..." + +curl http://127.0.0.1:4040/v1/systemone \ + -H 'content-type: application/json' \ + -d '{ + "model": "jev-latest", + "state": {"candidate": "The answer is 42."}, + "questions": { + "correct": { + "type": "noul", + "instructions": "The candidate is correct." + }, + "quality": { + "type": "choice", + "instructions": "Choose the quality band.", + "criteria": {"high": null, "low": null} + } + } + }' +``` + +`TYPESAFE_API_KEY` is sent as `Authorization: Bearer `. For an +enterprise proxy, configure the endpoint and an already-rendered authorization +header in `config.toml`: + +```toml +[upstream] +typesafe_base_url = "https://typesafe-proxy.example/v1" +typesafe_auth_header = "Bearer proxy-secret" +typesafe_max_retries = 2 +typesafe_max_retry_delay_ms = 60000 +``` + +The equivalent environment overrides are `NEMO_RELAY_TYPESAFE_BASE_URL` and +`NEMO_RELAY_TYPESAFE_AUTH_HEADER`. Retry overrides are +`NEMO_RELAY_TYPESAFE_MAX_RETRIES` and +`NEMO_RELAY_TYPESAFE_MAX_RETRY_DELAY_MS`. Relay never records authorization +headers in request annotations or configuration previews. + +## Non-Streaming and Retries + +System One is explicitly non-streaming. A request containing a `stream` field, +including `false` or `null`, is rejected before Relay opens an upstream +request. This makes accidental use of `llm.stream_execute` fail closed. + +Direct gateways and daemon-managed workers retry TypeSafe `408`, `429`, and all +`5xx` responses, plus connection failures and response-header timeouts, up to +two times by default. The policy honors `retry-after-ms`, numeric +`Retry-After`, and HTTP-date `Retry-After` values up to 60 seconds. Otherwise it +uses exponential backoff from 500 milliseconds to 5 seconds with up to 25% +subtractive jitter. +Relay strips caller-provided `x-typesafe-retry-count` and sets it on its own +retry attempts. + +The official SDKs also retry by default. To avoid multiplying attempts, choose +one retry owner for production: set the SDK policy to zero retries when Relay +owns retries, or set `typesafe_max_retries = 0` when the SDK owns them. + +## Evaluation Annotation + +`TypeSafeSystemOneCodec` leaves chat-only fields empty. Its request and response +annotations use a custom envelope with the following stable shape: + +```json +{ + "api_name": "typesafe.system_one", + "data": { + "provider": "typesafe", + "operation": "system_one", + "value": { + "model": "jev-latest", + "state": {}, + "questions": {} + } + } +} +``` + +Response `value` contains typed `answers` and `usage`, including +`billing_units` when TypeSafe reports it. Normalized Relay usage maps +`input_tokens` to prompt tokens and `output_tokens` to completion tokens. + +The GenAI OpenTelemetry projection records standard model and token attributes, +uses `gen_ai.operation.name=evaluate`, and adds: + +- `nemo_relay.evaluation.provider` +- `nemo_relay.evaluation.operation` +- `nemo_relay.evaluation.question_count` +- `nemo_relay.evaluation.answer_count` +- content-free counts for boolean, choice, and score answers + +Raw questions and answers remain in Relay lifecycle events, where event +sanitizers and redaction plugins can process them. They are deliberately not +copied into OTLP span attributes. + +## Pricing and Model Aliases + +Pricing remains catalog-driven. The following entry maps the moving +`jev-latest` alias to a pinned model ID. Replace the example pinned ID with the +exact model version returned by your account. + +```toml +version = 1 + +[[components]] +kind = "pricing" +enabled = true + +[[components.config.sources]] +type = "inline" + +[components.config.sources.catalog] +version = 1 + +[[components.config.sources.catalog.entries]] +provider = "typesafe" +model_id = "jev-1.13.0" +aliases = ["jev-latest"] +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-09-17" +pricing_source = "https://typesafe.ai/blog/introducing-system-one-models-and-jev" + +[components.config.sources.catalog.entries.rates] +input_per_million = 0.042 +output_per_million = 0.0 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" +``` + +TypeSafe's launch announcement lists `$0.042` per million input tokens and free +output tokens. Keep the catalog date and source because pricing can change. + +## Language Bindings + +- Rust: `nemo_relay::codec::typesafe_system_one::TypeSafeSystemOneCodec` and + the provider-neutral types under `nemo_relay::evaluation` +- Python: `nemo_relay.codecs.TypeSafeSystemOneCodec` +- Node.js: `TypeSafeSystemOneCodec` +- C: `nemo_relay_typesafe_system_one_codec_new()` + +All four bindings expose request decode, lossless encode, and response decode. + +## Provider-neutral evaluation lifecycle + +Gateway compatibility continues to use Relay's managed provider transport, but +it does not pretend Jev produced a chat completion: messages, text output, tool +calls, and finish reasons remain empty. Native integrations can use +`nemo_relay::api::evaluation::evaluation_call`, `evaluation_call_end`, or +`evaluation_execute`. These APIs emit `evaluator` scopes with provider-neutral +`EvaluationRequest` and `EvaluationResponse` payloads, independently of the +TypeSafe wire codec. + +Legacy NeMo Guardrails, adaptive prompt mutation, streaming replay, and +chat-trajectory PII projection explicitly decline this evaluation surface. +Evaluation-aware policy components should operate on state, questions, and +answers through the provider-neutral schema instead. diff --git a/docs/nemo-relay-cli/basic-usage.mdx b/docs/nemo-relay-cli/basic-usage.mdx index 200088757..5bac1772d 100644 --- a/docs/nemo-relay-cli/basic-usage.mdx +++ b/docs/nemo-relay-cli/basic-usage.mdx @@ -247,8 +247,9 @@ configuration. The upstream authorization-header controls show only whether a value is configured and never print it in menus or previews. Prefer -`NEMO_RELAY_OPENAI_AUTH_HEADER` and `NEMO_RELAY_ANTHROPIC_AUTH_HEADER` instead -of storing credentials in `config.toml`. +`NEMO_RELAY_OPENAI_AUTH_HEADER`, `NEMO_RELAY_ANTHROPIC_AUTH_HEADER`, and +`NEMO_RELAY_TYPESAFE_AUTH_HEADER` instead of storing credentials in +`config.toml`. ### Provider Upstreams @@ -259,6 +260,7 @@ compatible requests to a proxy, enterprise endpoint, or another provider host: [upstream] openai_base_url = "https://api.openai.com/v1" anthropic_base_url = "https://api.anthropic.com" +typesafe_base_url = "https://api.typesafe.ai/v1" ``` Relay normally chooses the upstream from the request path, regardless of which @@ -271,6 +273,8 @@ follows: | `/chat/completions`, `/v1/chat/completions` | `openai_base_url` | | `/models`, `/v1/models` | `openai_base_url` | | `/v1/messages`, `/v1/messages/count_tokens` | `anthropic_base_url` | +| `/systemone`, `/v1/systemone` | `typesafe_base_url` | +| `/typesafe/v1/systemone`, `/typesafe/v1/models` | `typesafe_base_url` | For ordinary provider requests, Relay resolves credentials in this order: @@ -279,10 +283,11 @@ For ordinary provider requests, Relay resolves credentials in this order: 2. If no inbound credential is present, Relay uses the route-specific custom `Authorization` value from `[upstream]` or its environment override. 3. If no custom value is configured, Relay reads `OPENAI_API_KEY` or - `ANTHROPIC_API_KEY` and applies the provider's standard authentication - scheme. + `ANTHROPIC_API_KEY`, or `TYPESAFE_API_KEY` and applies the provider's + standard authentication scheme. -Set `openai_auth_header` or `anthropic_auth_header` under `[upstream]` only when +Set `openai_auth_header`, `anthropic_auth_header`, or `typesafe_auth_header` +under `[upstream]` only when an enterprise gateway, proxy, or compatible endpoint requires a complete `Authorization` value such as `Bearer ...` or `Basic ...`. Prefer the corresponding environment variables so you don't store credentials in diff --git a/python/nemo_relay/_native.pyi b/python/nemo_relay/_native.pyi index e055075c0..c2ef9649e 100644 --- a/python/nemo_relay/_native.pyi +++ b/python/nemo_relay/_native.pyi @@ -1459,6 +1459,22 @@ class GeminiGenerateContentCodec: """Decode a Gemini response into a normalized response view.""" ... +class TypeSafeSystemOneCodec: + """Built-in codec for non-streaming TypeSafe System One evaluations.""" + + def __init__(self) -> None: + """Create a TypeSafe System One codec.""" + ... + def decode(self, request: LLMRequest) -> AnnotatedLLMRequest: + """Decode a System One request into normalized evaluation data.""" + ... + def encode(self, annotated: AnnotatedLLMRequest, original: LLMRequest) -> LLMRequest: + """Encode normalized evaluation changes into System One format.""" + ... + def decode_response(self, response: _Json) -> AnnotatedLLMResponse: + """Decode a System One response, including decisions and usage.""" + ... + class AdaptiveRuntime: """Hosted adaptive runtime bridge implemented by the native extension. diff --git a/python/nemo_relay/codecs.py b/python/nemo_relay/codecs.py index 7972c8a75..95c3806c7 100644 --- a/python/nemo_relay/codecs.py +++ b/python/nemo_relay/codecs.py @@ -52,6 +52,7 @@ async def impl(request: LLMRequest): OCIGenAIChatCodec, OpenAIChatCodec, OpenAIResponsesCodec, + TypeSafeSystemOneCodec, ) if TYPE_CHECKING: @@ -169,4 +170,5 @@ def decode_response(self, response: Json) -> "AnnotatedLLMResponse": "OCIGenAIChatCodec", "OpenAIChatCodec", "OpenAIResponsesCodec", + "TypeSafeSystemOneCodec", ] diff --git a/python/nemo_relay/codecs.pyi b/python/nemo_relay/codecs.pyi index 80b3496a1..49057be62 100644 --- a/python/nemo_relay/codecs.pyi +++ b/python/nemo_relay/codecs.pyi @@ -239,6 +239,20 @@ class GeminiGenerateContentCodec: """ ... +class TypeSafeSystemOneCodec: + """Built-in codec for non-streaming TypeSafe System One evaluations.""" + + def __init__(self) -> None: ... + def decode(self, request: LLMRequest) -> AnnotatedLLMRequest: + """Decode a System One request into normalized evaluation data.""" + ... + def encode(self, annotated: AnnotatedLLMRequest, original: LLMRequest) -> LLMRequest: + """Encode normalized evaluation changes into System One format.""" + ... + def decode_response(self, response: Json) -> AnnotatedLLMResponse: + """Decode a System One response, including decisions and usage.""" + ... + __all__ = [ "AnnotatedLLMRequest", "AnthropicMessagesCodec", @@ -248,4 +262,5 @@ __all__ = [ "OCIGenAIChatCodec", "OpenAIChatCodec", "OpenAIResponsesCodec", + "TypeSafeSystemOneCodec", ] diff --git a/python/tests/test_builtin_codecs.py b/python/tests/test_builtin_codecs.py index 8e5ec8edd..caf89b569 100644 --- a/python/tests/test_builtin_codecs.py +++ b/python/tests/test_builtin_codecs.py @@ -13,6 +13,8 @@ from typing import cast +import pytest + import nemo_relay from nemo_relay import ( AnnotatedLLMRequest, @@ -29,6 +31,7 @@ OCIGenAIChatCodec, OpenAIChatCodec, OpenAIResponsesCodec, + TypeSafeSystemOneCodec, ) # --------------------------------------------------------------------------- @@ -97,6 +100,12 @@ def test_gemini_codec_has_methods(self) -> None: assert hasattr(codec, "encode") assert hasattr(codec, "decode_response") + def test_typesafe_system_one_codec_constructable(self) -> None: + codec = TypeSafeSystemOneCodec() + assert hasattr(codec, "decode") + assert hasattr(codec, "encode") + assert hasattr(codec, "decode_response") + # --------------------------------------------------------------------------- # 2. Built-in codec decode/encode round-trip @@ -104,6 +113,51 @@ def test_gemini_codec_has_methods(self) -> None: class TestBuiltinCodecDecodeEncode: + def test_typesafe_system_one_round_trip_and_response(self) -> None: + codec = TypeSafeSystemOneCodec() + request = LLMRequest( + {}, + { + "model": "jev-latest", + "state": {"candidate": "42"}, + "questions": { + "correct": { + "type": "noul", + "instructions": "Is it correct?", + } + }, + "request_id": "preserved", + }, + ) + annotated = codec.decode(request) + assert annotated.messages == [] + assert codec.encode(annotated, request).content == request.content + + response = codec.decode_response( + { + "model": "jev-1.13.0", + "answers": {"correct": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 12, "output_tokens": 0}, + } + ) + assert response.model == "jev-1.13.0" + assert response.usage is not None + assert response.usage["prompt_tokens"] == 12 + + def test_typesafe_system_one_rejects_streaming(self) -> None: + codec = TypeSafeSystemOneCodec() + request = LLMRequest( + {}, + { + "model": "jev-latest", + "state": "candidate", + "questions": {"correct": {"type": "noul", "instructions": "Correct?"}}, + "stream": False, + }, + ) + with pytest.raises(RuntimeError, match="does not support streaming"): + codec.decode(request) + def test_openai_chat_decode(self) -> None: """OpenAIChatCodec.decode() returns AnnotatedLLMRequest.""" codec = OpenAIChatCodec()