From b85e1e2eb63460b49c118ba274067c2514a927e9 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 17:32:38 -0600 Subject: [PATCH] refactor(switchyard): remove built-in service integration Signed-off-by: Bryan Bednarski --- .github/workflows/ci.yaml | 1 - ATTRIBUTIONS-Rust.md | 422 ---- Cargo.lock | 40 - Cargo.toml | 3 - RELEASING.md | 15 +- crates/cli/Cargo.toml | 7 - crates/cli/src/plugins/editor_model.rs | 125 +- crates/cli/src/plugins/mod.rs | 5 - crates/cli/src/server/mod.rs | 53 +- .../tests/coverage/shared/plugins_tests.rs | 18 + crates/cli/tests/switchyard_process_e2e.rs | 367 ---- crates/core/src/error.rs | 2 +- crates/switchyard/Cargo.toml | 34 - crates/switchyard/README.md | 125 -- crates/switchyard/src/component.rs | 1819 ---------------- crates/switchyard/src/contract.rs | 187 -- crates/switchyard/src/lib.rs | 17 - crates/switchyard/src/stream_translation.rs | 94 - crates/switchyard/src/translation.rs | 316 --- crates/switchyard/tests/contract.rs | 79 - .../switchyard/tests/unit/component_tests.rs | 1827 ----------------- docs/about-nemo-relay/release-notes/index.mdx | 5 +- docs/configure-plugins/about.mdx | 5 +- docs/configure-plugins/switchyard/about.mdx | 242 +-- .../switchyard/configuration.mdx | 358 ---- docs/index.yml | 2 +- examples/switchyard/README.md | 114 - examples/switchyard/e2e-common.sh | 84 - examples/switchyard/fake_upstream.py | 85 - .../switchyard/hermes-ollama-plugins.toml | 102 - .../switchyard/hermes-ollama-profiles.yaml | 45 - examples/switchyard/otel-collector.yaml | 22 - examples/switchyard/plugins.toml | 75 - examples/switchyard/real-e2e-plugins.toml | 73 - examples/switchyard/real-e2e-profiles.yaml | 21 - .../switchyard/run-hermes-ollama-smoke.sh | 409 ---- examples/switchyard/run-real-e2e.sh | 101 - justfile | 7 - 38 files changed, 119 insertions(+), 7187 deletions(-) delete mode 100644 crates/cli/tests/switchyard_process_e2e.rs delete mode 100644 crates/switchyard/Cargo.toml delete mode 100644 crates/switchyard/README.md delete mode 100644 crates/switchyard/src/component.rs delete mode 100644 crates/switchyard/src/contract.rs delete mode 100644 crates/switchyard/src/lib.rs delete mode 100644 crates/switchyard/src/stream_translation.rs delete mode 100644 crates/switchyard/src/translation.rs delete mode 100644 crates/switchyard/tests/contract.rs delete mode 100644 crates/switchyard/tests/unit/component_tests.rs delete mode 100644 docs/configure-plugins/switchyard/configuration.mdx delete mode 100644 examples/switchyard/README.md delete mode 100755 examples/switchyard/e2e-common.sh delete mode 100644 examples/switchyard/fake_upstream.py delete mode 100644 examples/switchyard/hermes-ollama-plugins.toml delete mode 100644 examples/switchyard/hermes-ollama-profiles.yaml delete mode 100644 examples/switchyard/otel-collector.yaml delete mode 100644 examples/switchyard/plugins.toml delete mode 100644 examples/switchyard/real-e2e-plugins.toml delete mode 100644 examples/switchyard/real-e2e-profiles.yaml delete mode 100755 examples/switchyard/run-hermes-ollama-smoke.sh delete mode 100755 examples/switchyard/run-real-e2e.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 368518e3d..1784edabe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -413,7 +413,6 @@ jobs: nemo-relay nemo-relay-adaptive nemo-relay-pii-redaction - nemo-relay-switchyard nemo-relay-ffi nemo-relay-cli ) diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index a9df3ef75..9fa8457da 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -45389,428 +45389,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` -## switchyard-protocol - 0.1.0 -**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard -**License Type(s)**: Apache-2.0 -### License: https://spdx.org/licenses/Apache-2.0.html -``` -Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - -## switchyard-translation - 0.1.0 -**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard -**License Type(s)**: Apache-2.0 -### License: https://spdx.org/licenses/Apache-2.0.html -``` -Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - ## syn - 2.0.117 **Repository URL**: https://github.com/dtolnay/syn **License Type(s)**: Apache-2.0 diff --git a/Cargo.lock b/Cargo.lock index 74cedc04a..42453fb90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1796,7 +1796,6 @@ dependencies = [ "nemo-relay", "nemo-relay-adaptive", "nemo-relay-pii-redaction", - "nemo-relay-switchyard", "opentelemetry", "opentelemetry_sdk", "percent-encoding", @@ -1907,23 +1906,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "nemo-relay-switchyard" -version = "0.7.0" -dependencies = [ - "async-stream", - "axum", - "futures-util", - "nemo-relay", - "reqwest 0.12.28", - "schemars", - "serde", - "serde_json", - "switchyard-translation", - "tokio", - "uuid", -] - [[package]] name = "nemo-relay-types" version = "0.7.0" @@ -3501,28 +3483,6 @@ dependencies = [ "sval_nested", ] -[[package]] -name = "switchyard-protocol" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80192a3665425e92b4e10df3bff748cc60180cd028ba207bd70fb95df6869d5" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "switchyard-translation" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c977a683d8921e1ee7f01373162e14cbbfac28ef2a87dde8441c1691b2fed9" -dependencies = [ - "serde", - "serde_json", - "switchyard-protocol", - "thiserror 2.0.18", -] - [[package]] name = "syn" version = "2.0.117" diff --git a/Cargo.toml b/Cargo.toml index c62ce5393..af85b0a6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ members = [ "crates/worker", "crates/adaptive", "crates/pii-redaction", - "crates/switchyard", "crates/cli", # Language Bindings "crates/python", @@ -34,8 +33,6 @@ nemo-relay-worker-proto = { version = "0.7.0", path = "crates/worker-proto" } nemo-relay-worker = { version = "0.7.0", path = "crates/worker" } nemo-relay-adaptive = { version = "0.7.0", path = "crates/adaptive" } nemo-relay-pii-redaction = { version = "0.7.0", path = "crates/pii-redaction" } -nemo-relay-switchyard = { version = "0.7.0", path = "crates/switchyard" } -switchyard-translation = "0.1.0" nemo-relay-ffi = { version = "0.7.0", path = "crates/ffi" } nemo-relay-cli = { version = "0.7.0", path = "crates/cli" } opentelemetry = { version = "0.32", default-features = false } diff --git a/RELEASING.md b/RELEASING.md index 55286e1ca..5d7127df3 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -30,7 +30,7 @@ The release pipeline publishes these package surfaces from a tag push: | Ecosystem | Published Surface | |---|---| -| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, `nemo-relay-cli` | +| crates.io | `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, `nemo-relay-ffi`, `nemo-relay-cli` | | PyPI | `nemo-relay`, `nemo-relay-plugin`, `nemo-relay-cli-bin` | | npm | `nemo-relay-node` and its seven platform packages, `nemo-relay-openclaw`, `nemo-relay-cli-bin`, and its seven platform packages | | GitHub Releases | CLI binaries, `nemo-relay` and `nemo-relay-cli-bin` wheels, CLI and Node npm tarballs, and checksums | @@ -53,8 +53,7 @@ NeMo Relay versions are anchored on the workspace SemVer in the repository root - The root `Cargo.toml` `workspace.dependencies` entries for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli` must + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli` must stay aligned with that same version. - `crates/node/package.json` carries the base npm version for the Node.js package. The repository-root `package-lock.json` carries the npm workspace @@ -139,7 +138,7 @@ Before you create a release tag, confirm the following: - crates.io trusted publishers for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, - `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` are + `nemo-relay-ffi`, and `nemo-relay-cli` are configured for the top-level [`.github/workflows/ci.yaml`](.github/workflows/ci.yaml) workflow - GitHub Actions `id-token: write` access is available for the top-level npm publish job @@ -174,8 +173,7 @@ The helper updates: 2. The root [`Cargo.toml`](Cargo.toml) `workspace.dependencies` versions for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli`. + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli`. 3. [`crates/node/package.json`](crates/node/package.json) and the `crates/node` entry in the root [`package-lock.json`](package-lock.json) to the same release version. @@ -283,7 +281,7 @@ The release pipeline then: runs `cargo publish --package` for `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, `nemo-relay-pii-redaction`, - `nemo-relay-switchyard`, `nemo-relay-ffi`, and `nemo-relay-cli` through + `nemo-relay-ffi`, and `nemo-relay-cli` through trusted publishing from the top-level workflow - `publish-python` downloads the `nemo-relay`, `nemo-relay-plugin`, and @@ -354,8 +352,7 @@ After the release is live, verify: 1. The `nemo-relay-types`, `nemo-relay-plugin`, `nemo-relay-worker-proto`, `nemo-relay-worker`, `nemo-relay`, `nemo-relay-adaptive`, - `nemo-relay-pii-redaction`, `nemo-relay-switchyard`, `nemo-relay-ffi`, and - `nemo-relay-cli` crates + `nemo-relay-pii-redaction`, `nemo-relay-ffi`, and `nemo-relay-cli` crates are visible on crates.io. 2. The `nemo-relay` and `nemo-relay-cli-bin` wheels are visible on PyPI, and `pip install "nemo-relay[cli]"` exposes `nemo-relay`. diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index e04210891..79444c067 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,13 +23,11 @@ workspace = true [features] default = ["atof-streaming"] atof-streaming = ["nemo-relay/atof-streaming"] -switchyard = ["dep:nemo-relay-switchyard"] [dependencies] nemo-relay = { workspace = true, features = ["guardrails-remote", "object-store", "worker-grpc"] } nemo-relay-adaptive = { workspace = true, features = ["redis-backend"] } nemo-relay-pii-redaction.workspace = true -nemo-relay-switchyard = { workspace = true, optional = true, features = ["schema"] } async-stream = "0.3" axum = "0.8" base64 = "0.22" @@ -83,8 +81,3 @@ syn = { version = "2", features = ["full", "visit"] } tempfile = "3" tokio = { version = "1", features = ["test-util"] } tower = { version = "0.5", features = ["util"] } - -[[test]] -name = "switchyard_process_e2e" -path = "tests/switchyard_process_e2e.rs" -required-features = ["switchyard"] diff --git a/crates/cli/src/plugins/editor_model.rs b/crates/cli/src/plugins/editor_model.rs index 5fa842616..26aad6cdc 100644 --- a/crates/cli/src/plugins/editor_model.rs +++ b/crates/cli/src/plugins/editor_model.rs @@ -14,8 +14,6 @@ use nemo_relay::plugins::nemo_guardrails::component::{ use nemo_relay_adaptive::AdaptiveConfig; use nemo_relay_adaptive::plugin_component::ADAPTIVE_PLUGIN_KIND; use nemo_relay_pii_redaction::component::{PII_REDACTION_PLUGIN_KIND, PiiRedactionConfig}; -#[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::{Map, Value, json}; @@ -40,8 +38,6 @@ pub(super) enum EditableComponent { Adaptive(Box>), NemoGuardrails(Box>), PiiRedaction(Box>), - #[cfg(feature = "switchyard")] - Switchyard(Box>), } impl EditableComponent { @@ -51,8 +47,6 @@ impl EditableComponent { Self::Adaptive(_) => "Adaptive", Self::NemoGuardrails(_) => "NeMo Guardrails", Self::PiiRedaction(_) => "PII Redaction", - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard Decision API", } } @@ -62,8 +56,6 @@ impl EditableComponent { Self::Adaptive(_) => AdaptiveConfig::editor_schema().fields, Self::NemoGuardrails(_) => NeMoGuardrailsConfig::editor_schema().fields, Self::PiiRedaction(_) => PiiRedactionConfig::editor_schema().fields, - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => SwitchyardConfig::editor_schema().fields, } } @@ -73,8 +65,6 @@ impl EditableComponent { Self::Adaptive(state) => state.enabled, Self::NemoGuardrails(state) => state.enabled, Self::PiiRedaction(state) => state.enabled, - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.enabled, } } @@ -84,8 +74,6 @@ impl EditableComponent { Self::Adaptive(state) => state.toggle_enabled(), Self::NemoGuardrails(state) => state.toggle_enabled(), Self::PiiRedaction(state) => state.toggle_enabled(), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.toggle_enabled(), } } @@ -95,8 +83,6 @@ impl EditableComponent { Self::Adaptive(state) => state.set_enabled(enabled), Self::NemoGuardrails(state) => state.set_enabled(enabled), Self::PiiRedaction(state) => state.set_enabled(enabled), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.set_enabled(enabled), } } @@ -106,8 +92,6 @@ impl EditableComponent { Self::Adaptive(state) => state.reset_enabled(), Self::NemoGuardrails(state) => state.reset_enabled(), Self::PiiRedaction(state) => state.reset_enabled(), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => state.reset_enabled(), } } @@ -117,8 +101,6 @@ impl EditableComponent { Self::Adaptive(state) => adaptive_summary(state), Self::NemoGuardrails(state) => nemo_guardrails_summary(state), Self::PiiRedaction(state) => pii_redaction_summary(state), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => switchyard_summary(state), } } @@ -132,10 +114,6 @@ impl EditableComponent { Self::PiiRedaction(state) => { config_field_configured(&state.config, field).unwrap_or(false) } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - config_field_configured(&state.config, field).unwrap_or(false) - } } } @@ -157,11 +135,6 @@ impl EditableComponent { reset_config_field(&mut state.config, field)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - reset_config_field(&mut state.config, field)?; - state.mark_config_touched(); - } } Ok(()) } @@ -187,11 +160,6 @@ impl EditableComponent { remove_struct_field(&mut state.config, field.name)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => { - remove_struct_field(&mut state.config, field.name)?; - state.mark_config_touched(); - } } Ok(true) } @@ -202,8 +170,6 @@ impl EditableComponent { Self::Adaptive(state) => store_adaptive_state(config, state), Self::NemoGuardrails(state) => store_nemo_guardrails_state(config, state), Self::PiiRedaction(state) => store_pii_redaction_state(config, state), - #[cfg(feature = "switchyard")] - Self::Switchyard(state) => store_switchyard_state(config, state), } } } @@ -227,21 +193,12 @@ pub(super) enum ComponentMenuAction { pub(super) fn editable_components( config: &PluginConfig, ) -> Result, CliError> { - let components = vec![ + Ok(vec![ EditableComponent::Observability(Box::new(component_observability_state(config)?)), EditableComponent::Adaptive(Box::new(component_adaptive_state(config)?)), EditableComponent::NemoGuardrails(Box::new(component_nemo_guardrails_state(config)?)), EditableComponent::PiiRedaction(Box::new(component_pii_redaction_state(config)?)), - ]; - #[cfg(feature = "switchyard")] - let components = { - let mut components = components; - components.push(EditableComponent::Switchyard(Box::new( - component_switchyard_state(config)?, - ))); - components - }; - Ok(components) + ]) } pub(super) fn plugin_menu_items( @@ -446,13 +403,6 @@ pub(super) fn component_pii_redaction_state( component_editor_state(config, PII_REDACTION_PLUGIN_KIND, false) } -#[cfg(feature = "switchyard")] -pub(super) fn component_switchyard_state( - config: &PluginConfig, -) -> Result, CliError> { - component_editor_state(config, SWITCHYARD_PLUGIN_KIND, false) -} - pub(super) fn store_observability_state( config: &mut PluginConfig, state: &ComponentEditorState, @@ -517,23 +467,6 @@ pub(super) fn store_pii_redaction_state( Ok(()) } -#[cfg(feature = "switchyard")] -pub(super) fn store_switchyard_state( - config: &mut PluginConfig, - state: &ComponentEditorState, -) -> Result<(), CliError> { - if state.should_store(state.config_touched || switchyard_configured(&state.config)) { - store_component_editor_config( - config, - SWITCHYARD_PLUGIN_KIND, - state.enabled, - switchyard_config_map(&state.config)?, - merge_switchyard_editor_config, - ); - } - Ok(()) -} - fn store_component_editor_config( config: &mut PluginConfig, kind: &str, @@ -882,24 +815,6 @@ pub(super) fn pii_redaction_config_map( } } -#[cfg(feature = "switchyard")] -pub(super) fn switchyard_config_map( - config: &SwitchyardConfig, -) -> Result, CliError> { - let value = serde_json::to_value(config).map_err(serde_error)?; - match value { - Value::Object(mut map) => { - if is_version_one(map.get("version")) { - map.remove("version"); - } - Ok(map) - } - _ => Err(CliError::Config( - "switchyard config must serialize to an object".into(), - )), - } -} - pub(super) fn merge_observability_editor_config( existing: &mut Map, edited: Map, @@ -957,22 +872,6 @@ pub(super) fn merge_pii_redaction_editor_config( ); } -#[cfg(feature = "switchyard")] -pub(super) fn merge_switchyard_editor_config( - existing: &mut Map, - edited: Map, -) { - if is_version_one(existing.get("version")) { - existing.remove("version"); - } - merge_known_editor_object( - existing, - edited, - &nested_editor_keys(SwitchyardConfig::editor_schema()), - SwitchyardConfig::editor_schema(), - ); -} - fn is_version_one(value: Option<&Value>) -> bool { value.and_then(Value::as_u64) == Some(1) } @@ -1171,23 +1070,3 @@ pub(super) fn pii_redaction_summary(state: &ComponentEditorState bool { - !config.decision_profile_id.is_empty() || !config.targets.is_empty() -} - -#[cfg(feature = "switchyard")] -pub(super) fn switchyard_summary(state: &ComponentEditorState) -> String { - let profile = if state.config.decision_profile_id.is_empty() { - "unconfigured" - } else { - state.config.decision_profile_id.as_str() - }; - format!( - "component {}, profile {}, targets {}", - if state.enabled { "enabled" } else { "disabled" }, - profile, - state.config.targets.len() - ) -} diff --git a/crates/cli/src/plugins/mod.rs b/crates/cli/src/plugins/mod.rs index 61b542e2d..e2f5899c2 100644 --- a/crates/cli/src/plugins/mod.rs +++ b/crates/cli/src/plugins/mod.rs @@ -380,11 +380,6 @@ fn edit_component_field( edit_config_field(theme, &mut state.config, field)?; state.mark_config_touched(); } - #[cfg(feature = "switchyard")] - EditableComponent::Switchyard(state) => { - edit_config_field(theme, &mut state.config, field)?; - state.mark_config_touched(); - } } Ok(()) } diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index a263f0a54..f1e32e5ce 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -28,10 +28,6 @@ use nemo_relay::plugin::{ }; use nemo_relay_adaptive::plugin_component::register_adaptive_component; use nemo_relay_pii_redaction::component::register_pii_redaction_component; -#[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{ - register_switchyard_component, validate_switchyard_atof_configuration, -}; use reqwest::Client; use serde_json::Value; use subtle::ConstantTimeEq; @@ -892,10 +888,7 @@ impl ServerPluginActivation { pub(crate) enum PluginComponentSetupError { Adaptive(String), PiiRedaction(String), - #[cfg(feature = "switchyard")] - Switchyard(String), - #[cfg(feature = "switchyard")] - SwitchyardAtof(String), + RemovedSwitchyard, } impl PluginComponentSetupError { @@ -903,10 +896,7 @@ impl PluginComponentSetupError { match self { Self::Adaptive(_) => "Adaptive plugin", Self::PiiRedaction(_) => "PII redaction plugin", - #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard plugin", - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(_) => "Switchyard ATOF", + Self::RemovedSwitchyard => "Switchyard migration", } } @@ -915,10 +905,12 @@ impl PluginComponentSetupError { Self::Adaptive(error) | Self::PiiRedaction(error) => { format!("registration failed: {error}") } - #[cfg(feature = "switchyard")] - Self::Switchyard(error) => format!("registration failed: {error}"), - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => error.clone(), + Self::RemovedSwitchyard => { + "the built-in Switchyard service integration was removed; install the \ + `nvidia.switchyard` native API v2 dynamic plugin and move this component's \ + settings under `[[plugins.dynamic]]`" + .into() + } } } } @@ -935,20 +927,18 @@ impl std::fmt::Display for PluginComponentSetupError { "PII redaction plugin registration failed: {error}" ) } - #[cfg(feature = "switchyard")] - Self::Switchyard(error) => { - write!(formatter, "Switchyard plugin registration failed: {error}") - } - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => { - write!(formatter, "Switchyard ATOF validation failed: {error}") - } + Self::RemovedSwitchyard => write!( + formatter, + "the built-in Switchyard service integration was removed; install the \ + `nvidia.switchyard` native API v2 dynamic plugin and move this component's \ + settings under `[[plugins.dynamic]]`" + ), } } } pub(crate) fn register_and_validate_plugin_components( - _plugin_config: &PluginConfig, + plugin_config: &PluginConfig, ) -> Vec { let mut errors = Vec::new(); if let Err(error) = register_adaptive_component() { @@ -957,13 +947,12 @@ pub(crate) fn register_and_validate_plugin_components( if let Err(error) = register_pii_redaction_component() { errors.push(PluginComponentSetupError::PiiRedaction(error.to_string())); } - #[cfg(feature = "switchyard")] - if let Err(error) = register_switchyard_component() { - errors.push(PluginComponentSetupError::Switchyard(error.to_string())); - } - #[cfg(feature = "switchyard")] - if let Err(error) = validate_switchyard_atof_configuration(_plugin_config) { - errors.push(PluginComponentSetupError::SwitchyardAtof(error)); + if plugin_config + .components + .iter() + .any(|component| component.kind == "switchyard") + { + errors.push(PluginComponentSetupError::RemovedSwitchyard); } errors } diff --git a/crates/cli/tests/coverage/shared/plugins_tests.rs b/crates/cli/tests/coverage/shared/plugins_tests.rs index cd6cc9abf..3f4275cef 100644 --- a/crates/cli/tests/coverage/shared/plugins_tests.rs +++ b/crates/cli/tests/coverage/shared/plugins_tests.rs @@ -2198,6 +2198,24 @@ fn validate_config_accepts_pii_redaction_component() { validate_config(&config).unwrap(); } +#[test] +fn validate_config_rejects_removed_switchyard_component_with_migration_guidance() { + let config = PluginConfig { + components: vec![PluginComponentSpec { + kind: "switchyard".to_string(), + enabled: true, + config: serde_json::Map::new(), + }], + ..PluginConfig::default() + }; + + let error = validate_config(&config).unwrap_err().to_string(); + + assert!(error.contains("built-in Switchyard service integration was removed")); + assert!(error.contains("nvidia.switchyard")); + assert!(error.contains("[[plugins.dynamic]]")); +} + #[test] fn validate_config_rejects_local_nemo_guardrails_request_defaults() { let config = PluginConfig { diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs deleted file mode 100644 index a9ba3397d..000000000 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ /dev/null @@ -1,367 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! CI-safe process-boundary coverage for the Switchyard plugin. - -use std::process::{Child, Command, Stdio}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use axum::body::Body; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; -use axum::response::Response; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use serde_json::{Value, json}; - -fn gateway_bin() -> &'static str { - env!("CARGO_BIN_EXE_nemo-relay") -} - -struct ChildGuard(Child); - -impl Drop for ChildGuard { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - -#[derive(Clone, Default)] -struct DecisionState { - requests: Arc>>, -} - -async fn decide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let call = { - let mut requests = state.requests.lock().unwrap(); - requests.push((headers, request)); - requests.len() - }; - if call == 4 { - return Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .body(Body::from("decision API unavailable")) - .unwrap(); - } - let body = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": format!("decision-{call}"), - "router": {"name": "fake-ci-router", "version": "1"}, - "route": { - "tier": "strong", - "target_model": "provider/selected", - "backend_id": "selected-chat", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.99, - "reason_code": "ci_fixture", - "reason_summary": "deterministic process E2E decision" - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn switchyard_health() -> Json { - Json(json!({"status": "ok"})) -} - -#[derive(Clone, Default)] -struct ProviderState { - requests: Arc>>, -} - -async fn provide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let stream = request["stream"].as_bool().unwrap_or(false); - let model = request["model"].as_str().unwrap_or("unknown").to_string(); - let malformed_response = !stream - && model == "provider/selected" - && headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response"); - state.requests.lock().unwrap().push((headers, request)); - if malformed_response { - return Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from("{invalid-provider-json")) - .unwrap(); - } - if stream { - let first = json!({ - "id": "chat-ci", "object": "chat.completion.chunk", "model": model, - "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}, "finish_reason": null}] - }); - let last = json!({ - "id": "chat-ci", "object": "chat.completion.chunk", "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5} - }); - let body = format!("data: {first}\n\ndata: {last}\n\ndata: [DONE]\n\n"); - return Response::builder() - .status(StatusCode::OK) - .header("content-type", "text/event-stream") - .body(Body::from(body)) - .unwrap(); - } - let body = json!({ - "id": "chat-ci", "object": "chat.completion", "model": model, - "choices": [{"index": 0, "message": {"role": "assistant", "content": format!("served by {model}")}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn start_server(router: Router) -> (String, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let task = tokio::spawn(async move { - axum::serve(listener, router).await.unwrap(); - }); - (format!("http://{address}"), task) -} - -fn unused_address() -> std::net::SocketAddr { - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - listener.local_addr().unwrap() -} - -async fn wait_for_gateway(client: &reqwest::Client, url: &str, child: &mut Child) { - for _ in 0..120 { - if let Some(status) = child.try_wait().unwrap() { - panic!("gateway exited before readiness with {status}"); - } - if client - .get(format!("{url}/healthz")) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - return; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - panic!("gateway did not become ready at {url}"); -} - -#[tokio::test(flavor = "multi_thread")] -async fn switchyard_plugin_routes_buffered_and_streaming_then_fails_open() { - let decision_state = DecisionState::default(); - let decision_requests = Arc::clone(&decision_state.requests); - let (decision_url, decision_task) = start_server( - Router::new() - .route("/v1/routing/decision", post(decide)) - .route("/health", get(switchyard_health)) - .with_state(decision_state), - ) - .await; - - let provider_state = ProviderState::default(); - let provider_requests = Arc::clone(&provider_state.requests); - let (provider_url, provider_task) = start_server( - Router::new() - .route("/v1/chat/completions", post(provide)) - .with_state(provider_state), - ) - .await; - - let temp = tempfile::tempdir().unwrap(); - let config_path = temp.path().join("plugins.toml"); - let config = format!( - r#"version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "{decision_url}/v1/routing/decision" -decision_profile_id = "ci-process-e2e" -request_materialization = "full_body" -context_mode = "payload_only" -decision_timeout_millis = 1000 -max_retries = 0 - -[components.config.default_targets] -openai_chat = "fallback-chat" -openai_responses = "fallback-responses" -anthropic_messages = "fallback-anthropic" - -[components.config.targets.selected-chat] -model = "provider/selected" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "{provider_url}" - -[components.config.targets.fallback-chat] -model = "provider/fallback" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "{provider_url}" - -[components.config.targets.fallback-responses] -model = "provider/fallback" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "{provider_url}" - -[components.config.targets.fallback-anthropic] -model = "provider/fallback" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "{provider_url}" -"# - ); - std::fs::write(&config_path, config).unwrap(); - - let address = unused_address(); - let gateway_url = format!("http://{address}"); - let stderr = std::fs::File::create(temp.path().join("gateway.log")).unwrap(); - let child = Command::new(gateway_bin()) - .arg("--plugin-config-path") - .arg(&config_path) - .arg("--bind") - .arg(address.to_string()) - .stdout(Stdio::null()) - .stderr(Stdio::from(stderr)) - .spawn() - .unwrap(); - let mut gateway = ChildGuard(child); - let client = reqwest::Client::new(); - wait_for_gateway(&client, &gateway_url, &mut gateway.0).await; - - let send_chat = |request_id: &'static str, stream: bool| { - client - .post(format!("{gateway_url}/v1/chat/completions")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", request_id) - .header( - "x-nemo-relay-internal-dispatch-url", - "http://attacker.invalid", - ) - .header("x-nemo-relay-internal-dispatch-route", "attacker-route") - .json(&json!({ - "model": "client/model", - "stream": stream, - "messages": [{"role": "user", "content": "process boundary test"}] - })) - .send() - }; - - let buffered = send_chat("buffered-request", false).await.unwrap(); - assert!(buffered.status().is_success()); - let buffered: Value = buffered.json().await.unwrap(); - assert_eq!(buffered["model"], "provider/selected"); - - let translated = client - .post(format!("{gateway_url}/v1/responses")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", "translated-request") - .json(&json!({ - "model": "client/model", - "stream": false, - "input": "process boundary response translation" - })) - .send() - .await - .unwrap(); - assert!(translated.status().is_success()); - let translated: Value = translated.json().await.unwrap(); - assert_eq!(translated["object"], "response"); - - let streaming = send_chat("stream-request", true).await.unwrap(); - assert!(streaming.status().is_success()); - let streaming = streaming.text().await.unwrap(); - assert!(streaming.contains("streamed")); - assert!(streaming.contains("[DONE]")); - - let fallback = send_chat("fallback-request", false).await.unwrap(); - assert!(fallback.status().is_success()); - let fallback: Value = fallback.json().await.unwrap(); - assert_eq!(fallback["model"], "provider/fallback"); - - let malformed = send_chat("malformed-response", false).await.unwrap(); - assert!(malformed.status().is_success()); - let malformed: Value = malformed.json().await.unwrap(); - assert_eq!(malformed["model"], "provider/fallback"); - - let decisions = decision_requests.lock().unwrap(); - assert_eq!(decisions.len(), 5); - for (headers, body) in decisions.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - assert_eq!( - headers - .get("x-nemo-relay-session-id") - .unwrap() - .to_str() - .unwrap(), - "ci-process-session" - ); - assert_eq!(body["schema_version"], "switchyard.routing_request.v1"); - assert_eq!(body["decision_profile"]["profile_id"], "ci-process-e2e"); - } - drop(decisions); - - let providers = provider_requests.lock().unwrap(); - let models = providers - .iter() - .map(|(_, body)| body["model"].as_str().unwrap()) - .collect::>(); - assert_eq!( - models, - vec![ - "provider/selected", - "provider/selected", - "provider/selected", - "provider/fallback", - "provider/selected", - "provider/fallback" - ] - ); - assert!(providers[1].1["messages"].is_array()); - assert!(providers[1].1.get("input").is_none()); - assert_eq!(providers[1].1["messages"][0]["role"], "user"); - assert_eq!( - providers[1].1["messages"][0]["content"], - "process boundary response translation" - ); - let malformed_models = providers - .iter() - .filter(|(headers, _)| { - headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response") - }) - .map(|(_, body)| body["model"].as_str().unwrap()) - .collect::>(); - assert_eq!( - malformed_models, - vec!["provider/selected", "provider/fallback"] - ); - for (headers, _) in providers.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - } - - decision_task.abort(); - provider_task.abort(); -} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 988e5fb89..5863bb0fd 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -48,7 +48,7 @@ pub struct UpstreamFailure { } impl UpstreamFailure { - /// Whether Switchyard may be consulted for another bounded provider attempt. + /// Whether another bounded provider attempt may be made. pub fn is_retryable(&self) -> bool { matches!( self.class, diff --git a/crates/switchyard/Cargo.toml b/crates/switchyard/Cargo.toml deleted file mode 100644 index 7e7466884..000000000 --- a/crates/switchyard/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "nemo-relay-switchyard" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -description = "First-party Switchyard Decision API routing plugin for NeMo Relay." -readme = "README.md" - -[lints] -workspace = true - -[features] -default = [] -schema = ["dep:schemars", "nemo-relay/schema"] - -[dependencies] -nemo-relay.workspace = true -serde = { version = "1", features = ["derive"] } -serde_json = "1" -switchyard-translation.workspace = true -schemars = { version = "0.8", optional = true } -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } -tokio = { version = "1", default-features = false, features = ["time"] } -uuid = { workspace = true, features = ["v7"] } -futures-util = "0.3" -async-stream = "0.3" - -[dev-dependencies] -axum = "0.8" -tokio = { version = "1", features = ["net", "rt", "macros", "sync", "test-util", "rt-multi-thread", "time"] } diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md deleted file mode 100644 index ba50b355f..000000000 --- a/crates/switchyard/README.md +++ /dev/null @@ -1,125 +0,0 @@ - - -[![License](https://img.shields.io/github/license/NVIDIA/NeMo-Relay)](https://github.com/NVIDIA/NeMo-Relay/blob/main/LICENSE) -[![GitHub](https://img.shields.io/badge/github-repo-blue?logo=github)](https://github.com/NVIDIA/NeMo-Relay/) -[![Release](https://img.shields.io/github/v/release/NVIDIA/NeMo-Relay?color=green)](https://github.com/NVIDIA/NeMo-Relay/releases) - -# NeMo Relay Switchyard Plugin - -`nemo-relay-switchyard` is NeMo Relay's experimental integration -with the [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) -Decision API. It adds routing-aware LLM execution intercepts to the Relay -runtime while preserving Relay ownership of provider credentials, target -bindings, dispatch, retries, fallbacks, and observability. - -NeMo Relay 0.6.0 uses a separately running Switchyard Decision API from the -`topic/nemo-relay-integration` branch. - -Install it from crates.io, or build it from the NeMo Relay source checkout with -the optional CLI feature while the Switchyard Decision API contract and -service/library boundary are still evolving. - -Use the plugin to: - -- **Route through Switchyard decisions**: Select an exact Relay-owned target - using a versioned Decision API contract. -- **Keep provider protocols stable**: Use Switchyard's translation library for - OpenAI Chat, OpenAI Responses, and Anthropic Messages request and response - translation. -- **Preserve Relay execution semantics**: Keep retries, trusted fallbacks, - credentials, streaming behavior, and optimization accounting in Relay. -- **Support staged rollout**: Run in enforce or observe-only mode with - explicit target bindings and protocol defaults. - -## Implementation and Runtime Behavior - -The plugin includes the following implementation and runtime behavior: - -- `SwitchyardConfig`: The typed plugin configuration contract. -- `SwitchyardRuntime`: Buffered and streaming routing intercepts. -- Decision and target validation for exact backend, model, protocol, and - endpoint bindings. -- ATOF-backed or payload-only routing context modes. -- Routing marks and model-routing optimization contributions for Relay's - cumulative accounting pipeline. -- Switchyard-owned protocol translation through the - `switchyard-translation` dependency. - -## Installation and Source Build - -Add the crate from crates.io: - -```bash -cargo add nemo-relay-switchyard -``` - -To build the optional CLI integration from a NeMo Relay source checkout: - -```bash -cargo build -p nemo-relay-cli --features switchyard -cargo test -p nemo-relay-switchyard -``` - -The resulting CLI includes the Switchyard component only when the `switchyard` -feature is enabled. A default Relay build does not include this experimental -integration. - -## Runtime Boundary - -The current integration calls Switchyard's HTTP Decision API at runtime. Relay -does not start or supervise the Switchyard service. For ATOF-backed profiles, -Switchyard also provides the `/v1/atof/events` ingestion and accumulator -runtime. The service must therefore be running before Relay activates the -plugin. Activation performs a bounded request to the service's `/health` -endpoint and fails if it does not return `{"status":"ok"}`. This requirement -applies to enforce and observe-only rollout modes. - -The current service setup is documented in -[`examples/switchyard/README.md`](../../examples/switchyard/README.md), including -the pinned topic-branch commit, local configuration, compatibility smoke test, -and trajectory workflow. - -Translation runs in-process through Switchyard's Rust translation library. - -## Configuration and Registration - -The CLI registers the component when built with `--features switchyard` and -accepts a `[[components]]` entry with `kind = "switchyard"`. A minimal -configuration selects the Decision API and trusted protocol defaults: - -```toml -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "my-profile" -context_mode = "payload_only" -request_materialization = "summary_only" - -[components.config.default_targets] -openai_chat = "my-openai-target" -openai_responses = "my-responses-target" -anthropic_messages = "my-anthropic-target" -``` - -For ATOF-backed profiles, configure an enabled Relay ATOF HTTP stream sink that -has a unique `name`, targets the Switchyard ingestion URL, and uses -environment-referenced authentication headers. Set `atof_endpoint_name` in the -Switchyard component to that name. Local ATOF JSONL output alone does not -populate the Switchyard accumulator. Keep provider and Decision API credentials -outside tracked configuration files. - -## Documentation - -For more information, refer to the following resources: - -- [Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about) -- [Switchyard configuration reference](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/configuration) -- [Switchyard integration examples](../../examples/switchyard/README.md) -- [Switchyard repository](https://github.com/NVIDIA-NeMo/Switchyard) diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs deleted file mode 100644 index e8d5fced8..000000000 --- a/crates/switchyard/src/component.rs +++ /dev/null @@ -1,1819 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Switchyard plugin configuration and Relay execution integration. - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; - -use futures_util::StreamExt; -use nemo_relay::api::event::{CategoryProfile, DataSchema, EventCategory}; -use nemo_relay::api::llm::LlmRequest; -use nemo_relay::api::optimization::record_llm_optimization_contribution; -use nemo_relay::api::runtime::{ - LlmExecutionFn, LlmJsonStream, LlmStreamExecutionFn, LlmStreamInner, -}; -use nemo_relay::api::scope::{EmitMarkEventParams, event}; -use nemo_relay::codec::optimization::{ - LlmOptimizationContribution, LlmOptimizationKind, LlmOptimizationModel, - LlmOptimizationModelTransition, -}; -use nemo_relay::error::{FlowError, Result as FlowResult}; -use nemo_relay::observability::atof::{AtofEndpointFieldNamePolicy, AtofEndpointTransport}; -use nemo_relay::plugin::{ - ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginConfig, PluginError, - PluginRegistrationContext, Result as PluginResult, deregister_plugin, register_plugin, -}; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value as Json, json}; -use uuid::Uuid; - -use crate::contract::{ - DecisionAttempt, DecisionProfile, ROUTING_DECISION_SCHEMA_VERSION, - ROUTING_REQUEST_SCHEMA_VERSION, RequestIdentity, RequestMaterialization, RequestProtocol, - RequestSummary, RoutingDecision, RoutingRequest, RoutingTarget, -}; -use crate::stream_translation::StreamTranscoder; -use crate::translation::{ - decode_request, encode_request, latest_user_prompt, recent_message_window, translate_response, - translation_engine, validate_portable_request, -}; - -/// Plugin kind used in Relay plugin configuration. -pub const SWITCHYARD_PLUGIN_KIND: &str = "switchyard"; - -const SWITCHYARD_HEALTH_PATH: &str = "/health"; -const SWITCHYARD_HEALTH_TIMEOUT: Duration = Duration::from_secs(2); -const SWITCHYARD_HEALTH_MAX_ATTEMPTS: usize = 3; -const SWITCHYARD_HEALTH_INITIAL_BACKOFF: Duration = Duration::from_millis(100); -const INTERNAL_DISPATCH_URL_HEADER: &str = "x-nemo-relay-internal-dispatch-url"; -const INTERNAL_DISPATCH_ROUTE_HEADER: &str = "x-nemo-relay-internal-dispatch-route"; -const INTERNAL_RETRY_AWARE_HEADER: &str = "x-nemo-relay-internal-retry-aware"; -const ROUTING_MARK_SCHEMA: &str = "switchyard.routing_mark"; -const ROUTING_CONTRIBUTION_SCHEMA: &str = "nvidia.switchyard.routing_optimization"; - -/// Supported provider wire protocols. -#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum WireProtocol { - /// OpenAI Chat Completions. - OpenaiChat, - /// OpenAI Responses. - OpenaiResponses, - /// Anthropic Messages. - AnthropicMessages, -} - -impl WireProtocol { - fn label(self) -> &'static str { - match self { - Self::OpenaiChat => "openai_chat", - Self::OpenaiResponses => "openai_responses", - Self::AnthropicMessages => "anthropic_messages", - } - } - - fn endpoint(self) -> &'static str { - match self { - Self::OpenaiChat => "/v1/chat/completions", - Self::OpenaiResponses => "/v1/responses", - Self::AnthropicMessages => "/v1/messages", - } - } - - fn from_call(name: &str, request: &LlmRequest) -> Option { - match name { - "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { - Some(Self::OpenaiChat) - } - "openai.responses" | "openai_responses" => Some(Self::OpenaiResponses), - "anthropic.messages" | "anthropic" | "anthropic_messages" => { - Some(Self::AnthropicMessages) - } - _ if request.content.get("input").is_some() => Some(Self::OpenaiResponses), - _ if request.content.get("system").is_some() => Some(Self::AnthropicMessages), - _ if request.content.get("messages").is_some() => Some(Self::OpenaiChat), - _ => None, - } - } -} - -/// Routing rollout mode. -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RoutingMode { - /// Apply Switchyard decisions. - #[default] - Enforce, - /// Record decisions but dispatch trusted defaults. - ObserveOnly, -} - -impl RoutingMode { - fn label(self) -> &'static str { - match self { - Self::Enforce => "enforce", - Self::ObserveOnly => "observe_only", - } - } -} - -/// Whether the selected Switchyard profile depends on ATOF-derived history. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum ContextMode { - /// The router uses only current request material. - PayloadOnly, - /// Stable identity and a configured ATOF endpoint are required. - AtofRequired, -} - -/// Exact Relay-owned backend binding for one Switchyard backend ID. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct TargetBinding { - /// Exact model expected in the Switchyard decision. - pub model: String, - /// Exact protocol expected in the Switchyard decision. - pub protocol: WireProtocol, - /// Exact endpoint expected in the Switchyard decision. - pub endpoint: String, - /// Relay-owned backend base URL. - pub base_url: String, - /// Static non-sensitive backend headers. - #[serde(default)] - pub headers: BTreeMap, - /// Backend headers resolved from environment variables. - #[serde(default)] - pub header_env: BTreeMap, -} - -/// Trusted fallback target IDs for each inbound protocol. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct ProtocolDefaults { - /// OpenAI Chat fallback target. - #[serde(default)] - pub openai_chat: String, - /// OpenAI Responses fallback target. - #[serde(default)] - pub openai_responses: String, - /// Anthropic Messages fallback target. - #[serde(default)] - pub anthropic_messages: String, -} - -impl ProtocolDefaults { - fn target(&self, protocol: WireProtocol) -> &str { - match protocol { - WireProtocol::OpenaiChat => &self.openai_chat, - WireProtocol::OpenaiResponses => &self.openai_responses, - WireProtocol::AnthropicMessages => &self.anthropic_messages, - } - } -} - -/// Versioned Switchyard plugin configuration. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -pub struct SwitchyardConfig { - /// Config schema version. - #[serde(default = "default_version")] - pub version: u32, - /// Enforce or observe-only rollout mode. - #[serde(default)] - pub mode: RoutingMode, - /// Execution-intercept priority. - #[serde(default)] - pub priority: i32, - /// Switchyard Decision API URL. - pub decision_api_url: String, - /// Switchyard profile ID. - pub decision_profile_id: String, - /// Current-request materialization. - pub request_materialization: RequestMaterialization, - /// Profile context requirement. - pub context_mode: ContextMode, - /// Decision call timeout. - #[serde(default = "default_decision_timeout_millis")] - pub decision_timeout_millis: u64, - /// Provider retries after the initial attempt. - #[serde(default = "default_max_retries")] - pub max_retries: u32, - /// Number of messages in recent-message materialization. - #[serde(default = "default_recent_message_count")] - pub recent_message_count: usize, - /// Static non-sensitive Decision API headers. - #[serde(default)] - pub decision_headers: BTreeMap, - /// Decision API headers resolved from environment variables. - #[serde(default)] - pub decision_header_env: BTreeMap, - /// Enabled inbound protocols. - #[serde(default = "default_enabled_protocols")] - pub enabled_inbound_profiles: BTreeSet, - /// Exact backend bindings keyed by Switchyard backend ID. - pub targets: BTreeMap, - /// Trusted per-protocol fallbacks. - pub default_targets: ProtocolDefaults, - /// Named observability ATOF endpoint used by history-backed profiles. - #[serde(default)] - pub atof_endpoint_name: Option, -} - -impl Default for SwitchyardConfig { - fn default() -> Self { - Self { - version: default_version(), - mode: RoutingMode::default(), - priority: 0, - decision_api_url: "http://127.0.0.1:8080/v1/routing/decision".into(), - decision_profile_id: String::new(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: default_decision_timeout_millis(), - max_retries: default_max_retries(), - recent_message_count: default_recent_message_count(), - decision_headers: BTreeMap::new(), - decision_header_env: BTreeMap::new(), - enabled_inbound_profiles: default_enabled_protocols(), - targets: BTreeMap::new(), - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: String::new(), - anthropic_messages: String::new(), - }, - atof_endpoint_name: None, - } - } -} - -nemo_relay::editor_config! { - impl SwitchyardConfig { - mode => { label: "Rollout mode", kind: Enum, values: ["enforce", "observe_only"] }, - priority => { label: "Intercept priority", kind: Integer }, - decision_api_url => { label: "Decision API URL", kind: String }, - decision_profile_id => { label: "Decision profile ID", kind: String }, - request_materialization => { - label: "Request materialization", - kind: Enum, - values: ["none", "summary_only", "latest_user_prompt", "recent_message_window", "annotated_request", "full_body"] - }, - context_mode => { label: "Context mode", kind: Enum, values: ["payload_only", "atof_required"] }, - decision_timeout_millis => { label: "Decision timeout (ms)", kind: Integer }, - max_retries => { label: "Maximum provider retries", kind: Integer }, - recent_message_count => { label: "Recent message count", kind: Integer }, - decision_headers => { label: "Decision API static headers", kind: StringMap }, - decision_header_env => { label: "Decision API environment headers", kind: StringMap }, - enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json }, - targets => { label: "Backend target bindings", kind: Json }, - default_targets => { label: "Trusted protocol defaults", kind: Json }, - atof_endpoint_name => { label: "ATOF endpoint name", kind: String, optional: true } - } -} - -impl From for PluginComponentSpec { - fn from(value: SwitchyardConfig) -> Self { - let Json::Object(config) = - serde_json::to_value(value).expect("Switchyard config should serialize to an object") - else { - unreachable!("Switchyard config must serialize to an object") - }; - Self { - kind: SWITCHYARD_PLUGIN_KIND.into(), - enabled: true, - config, - } - } -} - -fn default_version() -> u32 { - 1 -} -fn default_decision_timeout_millis() -> u64 { - 25 -} -fn default_max_retries() -> u32 { - 3 -} -fn default_recent_message_count() -> usize { - 8 -} -fn default_enabled_protocols() -> BTreeSet { - BTreeSet::from([ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ]) -} - -struct SwitchyardPlugin; - -impl Plugin for SwitchyardPlugin { - fn plugin_kind(&self) -> &str { - SWITCHYARD_PLUGIN_KIND - } - - fn allows_multiple_components(&self) -> bool { - false - } - - fn validate(&self, plugin_config: &Map) -> Vec { - match parse_config(plugin_config).and_then(SwitchyardRuntime::new) { - Ok(_) => Vec::new(), - Err(error) => vec![ConfigDiagnostic { - level: DiagnosticLevel::Error, - code: "switchyard.invalid_config".into(), - component: Some(SWITCHYARD_PLUGIN_KIND.into()), - field: None, - message: error, - }], - } - } - - fn register<'a>( - &'a self, - plugin_config: &Map, - ctx: &'a mut PluginRegistrationContext, - ) -> Pin> + Send + 'a>> { - let parsed = parse_config(plugin_config); - Box::pin(async move { - let runtime = Arc::new( - parsed - .and_then(SwitchyardRuntime::new) - .map_err(PluginError::InvalidConfig)?, - ); - runtime - .require_healthy_sidecar() - .await - .map_err(PluginError::RegistrationFailed)?; - let buffered = Arc::clone(&runtime); - let buffered_intercept: LlmExecutionFn = Arc::new(move |name, request, next| { - let runtime = Arc::clone(&buffered); - let name = name.to_string(); - Box::pin(async move { runtime.execute_buffered(&name, request, next).await }) - }); - ctx.register_llm_execution_intercept( - "decision", - runtime.config.priority, - buffered_intercept, - )?; - - let streaming = Arc::clone(&runtime); - let stream_intercept: LlmStreamExecutionFn = Arc::new(move |name, request, next| { - let runtime = Arc::clone(&streaming); - let name = name.to_string(); - Box::pin(async move { runtime.execute_stream(&name, request, next).await }) - }); - ctx.register_llm_stream_execution_intercept( - "decision_stream", - runtime.config.priority, - stream_intercept, - )?; - Ok(()) - }) - } -} - -/// Register the first-party Switchyard component kind. -pub fn register_switchyard_component() -> PluginResult<()> { - match register_plugin(Arc::new(SwitchyardPlugin)) { - Ok(()) => Ok(()), - Err(PluginError::RegistrationFailed(message)) if message.contains("already registered") => { - Ok(()) - } - Err(error) => Err(error), - } -} - -/// Deregister the first-party Switchyard component kind. -pub fn deregister_switchyard_component() -> bool { - deregister_plugin(SWITCHYARD_PLUGIN_KIND) -} - -/// Validate the cross-component ATOF requirement for enabled history-backed profiles. -pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<(), String> { - let Some(component) = config - .components - .iter() - .find(|component| component.enabled && component.kind == SWITCHYARD_PLUGIN_KIND) - else { - return Ok(()); - }; - let switchyard = parse_config(&component.config)?; - if switchyard.context_mode != ContextMode::AtofRequired { - return Ok(()); - } - let required_name = validate_atof_endpoint_name(switchyard.atof_endpoint_name.as_deref())? - .ok_or_else(|| { - "atof_required Switchyard profiles require atof_endpoint_name".to_string() - })?; - let observability = config - .components - .iter() - .find(|component| component.enabled && component.kind == "observability") - .ok_or_else(|| "atof_required Switchyard profiles require observability".to_string())?; - let sinks = observability - .config - .get("atof") - .filter(|atof| atof.get("enabled").and_then(Json::as_bool) == Some(true)) - .and_then(|atof| atof.get("sinks")) - .and_then(Json::as_array) - .ok_or_else(|| { - "atof_required Switchyard profiles require an enabled ATOF endpoint".to_string() - })?; - let matching_sinks = sinks - .iter() - .filter(|sink| { - sink.get("type").and_then(Json::as_str) == Some("stream") - && sink.get("name").and_then(Json::as_str) == Some(required_name) - }) - .collect::>(); - let endpoint = match matching_sinks.as_slice() { - [sink] => *sink, - [] => { - return Err(format!( - "atof_required Switchyard profile requires named ATOF endpoint {required_name:?}" - )); - } - _ => { - return Err(format!( - "ATOF endpoint name {required_name:?} must resolve to exactly one endpoint" - )); - } - }; - let transport = endpoint.get("transport").map_or_else( - || Some(AtofEndpointTransport::default()), - |value| value.as_str().and_then(AtofEndpointTransport::parse), - ); - if transport != Some(AtofEndpointTransport::HttpPost) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use transport = http_post" - )); - } - let field_name_policy = endpoint.get("field_name_policy").map_or_else( - || Some(AtofEndpointFieldNamePolicy::default()), - |value| value.as_str().and_then(AtofEndpointFieldNamePolicy::parse), - ); - if field_name_policy != Some(AtofEndpointFieldNamePolicy::Preserve) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use field_name_policy = preserve" - )); - } - if endpoint - .get("header_env") - .and_then(Json::as_object) - .is_none_or(Map::is_empty) - { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} authentication must use at least one environment-referenced header" - )); - } - Ok(()) -} - -fn parse_config(config: &Map) -> Result { - serde_json::from_value(Json::Object(config.clone())) - .map_err(|error| format!("invalid Switchyard plugin config: {error}")) -} - -struct SwitchyardRuntime { - config: SwitchyardConfig, - client: reqwest::Client, - target_headers: BTreeMap>, - translation: switchyard_translation::TranslationEngine, -} - -enum BufferedAttempt { - Complete(Json), - Retry((String, String)), - Fallback(&'static str), -} - -enum StreamAttempt { - Committed(LlmJsonStream), - Retry((String, String)), - Fallback(&'static str), -} - -struct StreamAttemptContext { - routing_request: RoutingRequest, - decision: RoutingDecision, - attempt: u32, - max_attempts: u32, -} - -fn provider_fallback_reason(error: &FlowError) -> &'static str { - if error_is_retryable(error) { - "retry_exhausted" - } else { - "non_retryable_provider_error" - } -} - -impl SwitchyardRuntime { - fn new(config: SwitchyardConfig) -> Result { - validate_config(&config)?; - let headers = resolve_headers(&config.decision_headers, &config.decision_header_env)?; - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(Duration::from_millis(config.decision_timeout_millis)) - .build() - .map_err(|error| format!("failed to build Decision API client: {error}"))?; - let target_headers = config - .targets - .iter() - .map(|(id, target)| { - let headers = resolve_json_headers(&target.headers, &target.header_env)?; - Ok((id.clone(), headers)) - }) - .collect::>()?; - Ok(Self { - config, - client, - target_headers, - translation: translation_engine(), - }) - } - - // Skip the portability guard when no configured target uses a different protocol: with no - // possible cross-protocol translation, provider-specific fields never need to be portable. - fn may_translate_protocol(&self, inbound: WireProtocol) -> bool { - self.config - .targets - .values() - .any(|target| target.protocol != inbound) - } - - async fn require_healthy_sidecar(&self) -> Result<(), String> { - let health_url = switchyard_health_url(&self.config.decision_api_url)?; - let client = reqwest::Client::builder() - .timeout(SWITCHYARD_HEALTH_TIMEOUT) - .build() - .map_err(|error| format!("failed to build Switchyard health client: {error}"))?; - let mut backoff = SWITCHYARD_HEALTH_INITIAL_BACKOFF; - let mut final_error = None; - for attempt in 1..=SWITCHYARD_HEALTH_MAX_ATTEMPTS { - match check_switchyard_health(&client, &health_url).await { - Ok(()) => return Ok(()), - Err(error) => final_error = Some(error), - } - if attempt < SWITCHYARD_HEALTH_MAX_ATTEMPTS { - tokio::time::sleep(backoff).await; - backoff *= 2; - } - } - Err(final_error.expect("at least one Switchyard health attempt is configured")) - } - - async fn execute_buffered( - &self, - name: &str, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, - ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { - return next(original).await; - }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { - return next(original).await; - } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_buffered( - inbound, - original, - next, - "unsupported_provider_extension", - ) - .await; - } - - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_buffered(inbound, original, next, "observe_only") - .await; - } - - let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; - for attempt in 1..=max_attempts { - match self - .buffered_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - BufferedAttempt::Complete(response) => return Ok(response), - BufferedAttempt::Retry(retry) => previous = Some(retry), - BufferedAttempt::Fallback(reason) => { - return self - .dispatch_fallback_buffered(inbound, original, next, reason) - .await; - } - } - } - unreachable!("routing attempt loop always returns") - } - - async fn buffered_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(BufferedAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - match next(routed).await { - Ok(response) => { - match translate_response(&self.translation, target_protocol, inbound, &response) { - Ok(response) => { - self.record_routing_contribution(&decision, attempt, true); - Ok(BufferedAttempt::Complete(response)) - } - Err(error) => { - self.emit_error( - Some(&routing_request), - attempt, - "response_translation", - &error.to_string(), - ); - Ok(BufferedAttempt::Fallback("translation_error")) - } - } - } - Err(error) if error_is_retryable(&error) && attempt < max_attempts => { - let retry_reason = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &retry_reason); - self.emit_retry(&routing_request, &decision, attempt, &retry_reason); - Ok(BufferedAttempt::Retry(( - decision.route.backend_id, - retry_reason, - ))) - } - Err(error) => { - let summary = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &summary); - Ok(BufferedAttempt::Fallback(provider_fallback_reason(&error))) - } - } - } - - async fn execute_stream( - &self, - name: &str, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, - ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { - return next(original).await; - }; - if !self.config.enabled_inbound_profiles.contains(&inbound) { - return next(original).await; - } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_stream(inbound, original, next, "unsupported_provider_extension") - .await; - } - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_stream(inbound, original, next, "observe_only") - .await; - } - - let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; - for attempt in 1..=max_attempts { - match self - .stream_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - StreamAttempt::Committed(stream) => return Ok(stream), - StreamAttempt::Retry(retry) => previous = Some(retry), - StreamAttempt::Fallback(reason) => { - return self - .dispatch_fallback_stream(inbound, original, next, reason) - .await; - } - } - } - unreachable!("stream routing attempt loop always returns") - } - - async fn stream_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmStreamExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(StreamAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - let context = StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - }; - match next(routed).await { - Ok(mut upstream) => { - let first = upstream.next().await; - Ok(self.classify_open_stream(inbound, target_protocol, context, upstream, first)) - } - Err(error) => Ok(self.classify_stream_setup_error(context, error)), - } - } - - fn classify_open_stream( - &self, - inbound: WireProtocol, - target_protocol: WireProtocol, - context: StreamAttemptContext, - upstream: LlmJsonStream, - first: Option>, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - match first { - Some(Ok(first)) => { - self.record_routing_contribution(&decision, attempt, true); - let committed = LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(first)), - upstream, - }); - let output = if target_protocol == inbound { - committed - } else { - translated_stream( - target_protocol, - inbound, - decision.route.target_model.clone(), - committed, - ) - }; - StreamAttempt::Committed(mark_terminal_stream( - output, - "provider_stream_committed", - self.config.mode.label(), - identity_metadata(&routing_request), - )) - } - Some(Err(error)) if error_is_retryable(&error) && attempt < max_attempts => self - .retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - provider_error_summary(&error), - ), - None if attempt < max_attempts => self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - "empty_stream".into(), - ), - Some(Err(error)) => { - let summary = provider_error_summary(&error); - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_open", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) - } - None => StreamAttempt::Fallback("empty_stream"), - } - } - - fn classify_stream_setup_error( - &self, - context: StreamAttemptContext, - error: FlowError, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - let summary = provider_error_summary(&error); - if error_is_retryable(&error) && attempt < max_attempts { - return self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_setup", - summary, - ); - } - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_setup", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) - } - - fn retry_stream_attempt( - &self, - routing_request: &RoutingRequest, - decision: RoutingDecision, - attempt: u32, - error_class: &str, - reason: String, - ) -> StreamAttempt { - if reason != "empty_stream" { - self.emit_error(Some(routing_request), attempt, error_class, &reason); - } - self.emit_retry(routing_request, &decision, attempt, &reason); - StreamAttempt::Retry((decision.route.backend_id, reason)) - } - - async fn decided_request( - &self, - inbound: WireProtocol, - original: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result<(RoutingRequest, RoutingDecision, LlmRequest), String> { - let request = self.routing_request(inbound, original, attempt, previous)?; - self.emit_requested(&request); - let started = Instant::now(); - let response = self - .client - .post(&self.config.decision_api_url) - .header("x-nemo-relay-session-id", &request.identity.session_id) - .json(&request) - .send() - .await - .map_err(|error| format!("Decision API request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format!("Decision API returned HTTP {status}: {body}")); - } - let decision = response - .json::() - .await - .map_err(|error| format!("Decision API returned invalid JSON: {error}"))?; - self.validate_decision(&decision)?; - if let Some(baseline) = decision.baseline_route.as_ref() - && let Err(error) = self.validate_target(baseline) - { - self.emit_error(Some(&request), attempt, "baseline_binding", &error); - } - let routed = self.apply_target(inbound, original.clone(), &decision)?; - let latency = started.elapsed().as_millis() as u64; - self.emit_decision( - &request, - &decision, - attempt, - self.config.mode == RoutingMode::ObserveOnly, - latency, - ); - Ok((request, decision, routed)) - } - - fn routing_request( - &self, - inbound: WireProtocol, - request: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result { - let session = header(request, "x-nemo-relay-session-id"); - let stable_request_id = header(request, "x-nemo-relay-request-id"); - if self.config.context_mode == ContextMode::AtofRequired - && (session.is_none() || stable_request_id.is_none()) - { - return Err("stable session and request identity are required for this profile".into()); - } - let identity_is_stable = session.is_some() && stable_request_id.is_some(); - let synthetic_session = format!("request-{}", Uuid::now_v7()); - let session_id = session.unwrap_or_else(|| synthetic_session.clone()); - let request_id = stable_request_id.unwrap_or_else(|| format!("request-{}", Uuid::now_v7())); - let annotated = decode_request(&self.translation, inbound, request) - .map_err(|error| format!("request translation decode failed: {error}"))?; - let current_request = self.materialize(inbound, request, &annotated)?; - let (previous_route, retry_reason) = previous.unzip(); - Ok(RoutingRequest { - schema_version: ROUTING_REQUEST_SCHEMA_VERSION.into(), - decision_profile: DecisionProfile { - profile_id: self.config.decision_profile_id.clone(), - request_materialization: self.config.request_materialization, - }, - identity: RequestIdentity { - session_id, - request_id, - turn_id: header(request, "x-nemo-relay-turn-id"), - parent_scope_id: header(request, "x-nemo-relay-parent-scope-id"), - root_scope_id: header(request, "x-nemo-relay-root-scope-id"), - harness: header(request, "x-nemo-relay-agent-kind") - .unwrap_or_else(|| "unknown".into()), - source: header(request, "x-nemo-relay-source") - .unwrap_or_else(|| "nemo-relay".into()), - owner_id: header(request, "x-nemo-relay-owner-id"), - quality: header(request, "x-nemo-relay-identity-quality").unwrap_or_else(|| { - if identity_is_stable { - "explicit".into() - } else { - "synthetic".into() - } - }), - }, - protocol: RequestProtocol { - inbound_profile: inbound.label().into(), - inbound_endpoint: inbound.endpoint().into(), - desired_response_profile: inbound.label().into(), - }, - request_summary: RequestSummary { - client_requested_model: request - .content - .get("model") - .and_then(Json::as_str) - .map(ToOwned::to_owned), - prompt_token_estimate: None, - tool_count_in_payload: request - .content - .get("tools") - .and_then(Json::as_array) - .map(|tools| tools.len() as u64), - has_system_prompt: Some( - annotated.instructions.iter().any(|instruction| { - instruction.role == switchyard_translation::Role::System - }) || annotated - .messages - .iter() - .any(|message| message.role == switchyard_translation::Role::System), - ), - }, - current_request, - attempt: DecisionAttempt { - routing_attempt: attempt, - max_routing_attempts: self.config.max_retries.saturating_add(1), - previous_route, - retry_reason, - }, - }) - } - - fn materialize( - &self, - inbound: WireProtocol, - request: &LlmRequest, - annotated: &switchyard_translation::LlmRequest, - ) -> Result, String> { - match self.config.request_materialization { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => Ok(None), - RequestMaterialization::FullBody => Ok(Some(json!({"body": request.content}))), - RequestMaterialization::AnnotatedRequest => Ok(Some(json!({ - "body": request.content, - "annotated_request": annotated, - }))), - RequestMaterialization::LatestUserPrompt => { - let prompt = latest_user_prompt(annotated) - .ok_or_else(|| "latest_user_prompt requires a user message".to_string())?; - let latest = recent_message_window(annotated, 1); - let body = encode_request(&self.translation, inbound, &latest, Map::new()) - .map_err(|error| format!("latest user prompt encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "latest_user_prompt": prompt}))) - } - RequestMaterialization::RecentMessageWindow => { - let window = recent_message_window(annotated, self.config.recent_message_count); - let body = encode_request(&self.translation, inbound, &window, Map::new()) - .map_err(|error| format!("recent window encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "annotated_request": window}))) - } - } - } - - fn validate_decision(&self, decision: &RoutingDecision) -> Result<(), String> { - if decision.schema_version != ROUTING_DECISION_SCHEMA_VERSION { - return Err(format!( - "unsupported decision schema {:?}", - decision.schema_version - )); - } - self.validate_target(&decision.route).map(|_| ()) - } - - fn validate_target(&self, target: &RoutingTarget) -> Result<&TargetBinding, String> { - let binding = self - .config - .targets - .get(&target.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", target.backend_id))?; - if binding.model != target.target_model - || binding.protocol.label() != target.target_protocol_profile - || binding.endpoint != target.target_endpoint - { - return Err(format!( - "decision target {:?} does not match its exact Relay binding", - target.backend_id - )); - } - Ok(binding) - } - - fn record_routing_contribution(&self, decision: &RoutingDecision, attempt: u32, applied: bool) { - let Some(contribution) = self.routing_contribution(decision, attempt, applied) else { - return; - }; - let _ = record_llm_optimization_contribution(contribution); - } - - fn routing_contribution( - &self, - decision: &RoutingDecision, - attempt: u32, - applied: bool, - ) -> Option { - let baseline = decision - .baseline_route - .as_ref() - .filter(|baseline| self.validate_target(baseline).is_ok())?; - let mut contribution = LlmOptimizationContribution::new( - SWITCHYARD_PLUGIN_KIND, - LlmOptimizationKind::model_routing(), - ); - contribution.applied = applied; - contribution.model_transition = Some(LlmOptimizationModelTransition { - baseline: Some(LlmOptimizationModel::new(&baseline.target_model)), - effective: Some(LlmOptimizationModel::new(&decision.route.target_model)), - }); - contribution.payload_schema = Some(DataSchema { - name: ROUTING_CONTRIBUTION_SCHEMA.to_string(), - version: "1".to_string(), - }); - contribution.payload = Some(json!({ - "decision_id": decision.decision_id, - "selected_backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "baseline_backend_id": baseline.backend_id, - "baseline_tier": baseline.tier, - "routing_attempt": attempt, - "rollout_mode": self.config.mode.label(), - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - })); - Some(contribution) - } - - fn apply_target( - &self, - inbound: WireProtocol, - request: LlmRequest, - decision: &RoutingDecision, - ) -> Result { - let binding = self - .config - .targets - .get(&decision.route.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", decision.route.backend_id))?; - let annotated = decode_request(&self.translation, inbound, &request) - .map_err(|error| format!("request decode failed: {error}"))?; - let mut routed = if inbound == binding.protocol { - request - } else { - encode_request( - &self.translation, - binding.protocol, - &annotated, - request.headers, - ) - .map_err(|error| format!("request translation failed: {error}"))? - }; - let object = routed - .content - .as_object_mut() - .ok_or_else(|| "translated request body is not an object".to_string())?; - object.insert("model".into(), Json::String(binding.model.clone())); - if let Some(headers) = self.target_headers.get(&decision.route.backend_id) { - routed.headers.extend(headers.clone()); - } - routed.headers.insert( - INTERNAL_DISPATCH_ROUTE_HEADER.into(), - Json::String(binding.protocol.label().into()), - ); - routed.headers.insert( - INTERNAL_DISPATCH_URL_HEADER.into(), - Json::String(dispatch_url(&binding.base_url, &binding.endpoint)), - ); - routed.headers.insert( - INTERNAL_RETRY_AWARE_HEADER.into(), - Json::String("true".into()), - ); - Ok(routed) - } - - fn fallback_request( - &self, - inbound: WireProtocol, - request: LlmRequest, - ) -> Result { - let id = self.config.default_targets.target(inbound); - let binding = self - .config - .targets - .get(id) - .ok_or_else(|| format!("unknown fallback target {id:?}"))?; - let decision = RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "relay-fallback".into(), - router: crate::contract::DecisionProvider { - name: "relay-fallback".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "fallback".into(), - target_model: binding.model.clone(), - backend_id: id.to_string(), - target_protocol_profile: binding.protocol.label().into(), - target_endpoint: binding.endpoint.clone(), - }, - baseline_route: None, - confidence: None, - reason_code: Some("relay_trusted_fallback".into()), - reason_summary: None, - metadata: BTreeMap::new(), - extra: BTreeMap::new(), - }; - self.apply_target(inbound, request, &decision) - } - - async fn dispatch_fallback_buffered( - &self, - inbound: WireProtocol, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, - reason: &str, - ) -> FlowResult { - self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(response) => Ok(response), - Err(error) => { - emit_terminal_error( - &error, - "fallback_buffered", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } - } - - async fn dispatch_fallback_stream( - &self, - inbound: WireProtocol, - original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, - reason: &str, - ) -> FlowResult { - self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(stream) => Ok(mark_terminal_stream( - stream, - "fallback_stream", - self.config.mode.label(), - metadata.clone(), - )), - Err(error) => { - emit_terminal_error( - &error, - "fallback_stream_setup", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } - } - - fn emit_requested(&self, request: &RoutingRequest) { - emit_mark( - "switchyard.routing.requested", - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "routing_attempt": request.attempt.routing_attempt, - "profile_id": request.decision_profile.profile_id, - "rollout_mode": self.config.mode.label(), - }), - identity_metadata(request), - ); - } - - fn emit_decision( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - observe_only: bool, - latency_ms: u64, - ) { - emit_mark( - "switchyard.routing.decision", - json!({ - "decision_id": decision.decision_id, - "profile_id": request.decision_profile.profile_id, - "router": decision.router.name, - "router_version": decision.router.version, - "routing_attempt": attempt, - "backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "selected_model": decision.route.target_model, - "target_protocol_profile": decision.route.target_protocol_profile, - "target_endpoint": decision.route.target_endpoint, - "confidence": decision.confidence, - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - "latency_ms": latency_ms, - "observe_only": observe_only, - "rollout_mode": self.config.mode.label(), - }), - identity_metadata(request), - ); - } - - fn emit_retry( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - reason: &str, - ) { - emit_mark( - "switchyard.routing.retry", - json!({"routing_attempt": attempt, "previous_route": decision.route.backend_id, "retry_reason": reason, "rollout_mode": self.config.mode.label()}), - identity_metadata(request), - ); - } - - fn emit_error(&self, request: Option<&RoutingRequest>, attempt: u32, class: &str, error: &str) { - emit_mark( - "switchyard.routing.error", - json!({"routing_attempt": attempt, "error_class": class, "error": error, "rollout_mode": self.config.mode.label()}), - request.map(identity_metadata).unwrap_or_else(|| json!({})), - ); - } - - fn emit_fallback(&self, inbound: WireProtocol, reason: &str, request: &LlmRequest) { - emit_mark( - "switchyard.routing.fallback", - json!({ - "fallback_reason": reason, - "fallback_route": self.config.default_targets.target(inbound), - "inbound_profile": inbound.label(), - "rollout_mode": self.config.mode.label(), - }), - identity_metadata_from_request(request), - ); - } -} - -async fn check_switchyard_health( - client: &reqwest::Client, - health_url: &reqwest::Url, -) -> Result<(), String> { - let response = client - .get(health_url.clone()) - .send() - .await - .map_err(|error| { - format!("Switchyard service is required but health check {health_url} failed: {error}") - })?; - let status = response.status(); - if !status.is_success() { - return Err(format!( - "Switchyard service is required but health check {health_url} returned HTTP {status}" - )); - } - let body = response.json::().await.map_err(|error| { - format!("Switchyard health check {health_url} returned invalid JSON: {error}") - })?; - if body.get("status").and_then(Json::as_str) != Some("ok") { - return Err(format!( - "Switchyard health check {health_url} did not report status=ok" - )); - } - Ok(()) -} - -fn switchyard_health_url(decision_api_url: &str) -> Result { - let mut url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - url.set_path(SWITCHYARD_HEALTH_PATH); - url.set_query(None); - url.set_fragment(None); - Ok(url) -} - -fn validate_atof_endpoint_name(name: Option<&str>) -> Result, String> { - if let Some(name) = name { - if name.trim().is_empty() { - return Err("atof_endpoint_name must be non-empty when configured".into()); - } - if name != name.trim() { - return Err("atof_endpoint_name must not have leading or trailing whitespace".into()); - } - } - Ok(name) -} - -fn validate_config(config: &SwitchyardConfig) -> Result<(), String> { - validate_scalar_config(config)?; - validate_decision_api_url(&config.decision_api_url)?; - validate_target_bindings(config)?; - validate_default_targets(config) -} - -fn validate_scalar_config(config: &SwitchyardConfig) -> Result<(), String> { - if config.version != 1 { - return Err(format!( - "unsupported Switchyard config version {}", - config.version - )); - } - if config.decision_profile_id.trim().is_empty() { - return Err("decision_profile_id must be non-empty".into()); - } - if config.decision_timeout_millis == 0 { - return Err("decision_timeout_millis must be greater than zero".into()); - } - if config.max_retries > 10 { - return Err("max_retries must not exceed 10".into()); - } - if config.recent_message_count == 0 { - return Err("recent_message_count must be greater than zero".into()); - } - let atof_endpoint_name = validate_atof_endpoint_name(config.atof_endpoint_name.as_deref())?; - if config.context_mode == ContextMode::AtofRequired && atof_endpoint_name.is_none() { - return Err("atof_required Switchyard profiles require atof_endpoint_name".into()); - } - Ok(()) -} - -fn validate_decision_api_url(decision_api_url: &str) -> Result<(), String> { - let url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - if !matches!(url.scheme(), "http" | "https") { - return Err("decision_api_url must use http or https".into()); - } - Ok(()) -} - -fn validate_target_bindings(config: &SwitchyardConfig) -> Result<(), String> { - if config.targets.is_empty() { - return Err("targets must not be empty".into()); - } - if config.enabled_inbound_profiles.is_empty() { - return Err("enabled_inbound_profiles must not be empty".into()); - } - let mut exact_bindings = BTreeSet::new(); - for (id, target) in &config.targets { - if id.trim().is_empty() - || target.model.trim().is_empty() - || target.endpoint.trim().is_empty() - { - return Err("target IDs, models, and endpoints must be non-empty".into()); - } - let base_url = reqwest::Url::parse(&target.base_url) - .map_err(|error| format!("target {id:?} base_url is invalid: {error}"))?; - if !matches!(base_url.scheme(), "http" | "https") { - return Err(format!("target {id:?} base_url must use http or https")); - } - if target.endpoint != target.protocol.endpoint() { - return Err(format!( - "target {id:?} endpoint must be {:?} for {}", - target.protocol.endpoint(), - target.protocol.label() - )); - } - if !exact_bindings.insert(( - target.model.clone(), - target.protocol, - target.endpoint.clone(), - target.base_url.trim_end_matches('/').to_string(), - )) { - return Err(format!( - "target {id:?} conflicts with another exact backend binding" - )); - } - } - Ok(()) -} - -fn validate_default_targets(config: &SwitchyardConfig) -> Result<(), String> { - for &protocol in &config.enabled_inbound_profiles { - let id = config.default_targets.target(protocol); - let target = config - .targets - .get(id) - .ok_or_else(|| format!("default target {id:?} is not configured"))?; - if target.protocol != protocol { - return Err(format!( - "default target {id:?} must use protocol {}", - protocol.label() - )); - } - } - Ok(()) -} - -fn resolve_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result { - let mut headers = HeaderMap::new(); - for (name, value) in static_headers { - insert_http_header(&mut headers, name, value)?; - } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - insert_http_header(&mut headers, name, &value)?; - } - Ok(headers) -} - -fn resolve_json_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result, String> { - let mut headers = Map::new(); - for (name, value) in static_headers { - headers.insert(name.clone(), Json::String(value.clone())); - } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "target header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - headers.insert(name.clone(), Json::String(value)); - } - Ok(headers) -} - -fn insert_http_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), String> { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid header name: {error}"))?; - let value = - HeaderValue::from_str(value).map_err(|error| format!("invalid header value: {error}"))?; - headers.insert(name, value); - Ok(()) -} - -fn protocol_from_label(label: &str) -> FlowResult { - match label { - "openai_chat" | "openai_chat_completions" | "openai_chat_completions.v1" => { - Ok(WireProtocol::OpenaiChat) - } - "openai_responses" | "openai_responses.v1" => Ok(WireProtocol::OpenaiResponses), - "anthropic_messages" | "anthropic_messages.v1" => Ok(WireProtocol::AnthropicMessages), - value => Err(FlowError::InvalidArgument(format!( - "unsupported Switchyard target protocol {value:?}" - ))), - } -} - -fn header(request: &LlmRequest, name: &str) -> Option { - request - .headers - .get(name) - .and_then(Json::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) -} - -fn dispatch_url(base_url: &str, endpoint: &str) -> String { - let base = base_url.trim_end_matches('/'); - let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { - &endpoint[3..] - } else { - endpoint - }; - format!("{base}{endpoint}") -} - -fn identity_metadata(request: &RoutingRequest) -> Json { - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "turn_id": request.identity.turn_id, - "owner_id": request.identity.owner_id, - }) -} - -fn identity_metadata_from_request(request: &LlmRequest) -> Json { - json!({ - "session_id": header(request, "x-nemo-relay-session-id"), - "request_id": header(request, "x-nemo-relay-request-id"), - "turn_id": header(request, "x-nemo-relay-turn-id"), - "owner_id": header(request, "x-nemo-relay-owner-id"), - }) -} - -fn error_is_retryable(error: &FlowError) -> bool { - matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) -} - -fn emit_mark(name: &str, data: Json, metadata: Json) { - if let Err(error) = event( - EmitMarkEventParams::builder() - .name(name) - .data(data) - .data_schema( - DataSchema::builder() - .name(ROUTING_MARK_SCHEMA) - .version("1") - .build(), - ) - .metadata(metadata) - .category(EventCategory::custom()) - .category_profile(CategoryProfile::builder().subtype(name).build()) - .build(), - ) { - eprintln!("nemo-relay switchyard: failed to emit {name}: {error}"); - } -} - -fn emit_terminal_error(error: &FlowError, phase: &str, rollout_mode: &str, metadata: Json) { - emit_mark( - "switchyard.routing.terminal_error", - json!({"error_class": provider_error_class(error), "error": provider_error_summary(error), "phase": phase, "rollout_mode": rollout_mode}), - metadata, - ); -} - -fn provider_error_class(error: &FlowError) -> &'static str { - match error { - FlowError::Upstream(failure) => match failure.class { - nemo_relay::error::UpstreamFailureClass::Connection => "connection", - nemo_relay::error::UpstreamFailureClass::Timeout => "timeout", - nemo_relay::error::UpstreamFailureClass::RetryableStatus => "retryable_status", - nemo_relay::error::UpstreamFailureClass::ContextWindow => "context_window", - nemo_relay::error::UpstreamFailureClass::ModelUnavailable => "model_unavailable", - nemo_relay::error::UpstreamFailureClass::Authentication => "authentication", - nemo_relay::error::UpstreamFailureClass::InvalidRequest => "invalid_request", - nemo_relay::error::UpstreamFailureClass::Other => "other", - }, - _ => "relay", - } -} - -fn provider_error_summary(error: &FlowError) -> String { - match error { - FlowError::Upstream(failure) => match failure.status { - Some(status) => format!("{}:http_{status}", provider_error_class(error)), - None => provider_error_class(error).to_string(), - }, - _ => error.to_string(), - } -} - -struct PrefixedStream { - first: Option>, - upstream: LlmJsonStream, -} - -impl futures_util::Stream for PrefixedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(first) = self.first.take() { - Poll::Ready(Some(first)) - } else { - Pin::new(&mut self.upstream).poll_next(cx) - } - } -} - -impl LlmStreamInner for PrefixedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.first = None; - Box::pin(async move { this.upstream.close().await }) - } -} - -struct TerminalMarkedStream { - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, - finished: bool, -} - -impl futures_util::Stream for TerminalMarkedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.finished { - return Poll::Ready(None); - } - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => Poll::Ready(Some(Ok(chunk))), - Poll::Ready(Some(Err(error))) => { - self.finished = true; - emit_terminal_error(&error, self.phase, self.rollout_mode, self.metadata.clone()); - Poll::Ready(Some(Err(error))) - } - Poll::Ready(None) => { - self.finished = true; - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } -} - -impl LlmStreamInner for TerminalMarkedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - Box::pin(async move { self.get_mut().upstream.close().await }) - } -} - -struct TranslatedStream { - upstream: LlmJsonStream, - transcoder: StreamTranscoder, - buffered: VecDeque>, - upstream_finished: bool, -} - -impl futures_util::Stream for TranslatedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - loop { - if let Some(chunk) = self.buffered.pop_front() { - return Poll::Ready(Some(chunk)); - } - if self.upstream_finished { - return Poll::Ready(None); - } - - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => match self.transcoder.transcode(&chunk) { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - }, - Poll::Ready(Some(Err(error))) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - Poll::Ready(None) => { - self.upstream_finished = true; - match self.transcoder.finish() { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => return Poll::Ready(Some(Err(error))), - } - } - Poll::Pending => return Poll::Pending, - } - } - } -} - -impl LlmStreamInner for TranslatedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.buffered.clear(); - this.upstream_finished = true; - Box::pin(async move { this.upstream.close().await }) - } -} - -fn mark_terminal_stream( - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TerminalMarkedStream { - upstream, - phase, - rollout_mode, - metadata, - finished: false, - }) -} - -fn translated_stream( - source: WireProtocol, - target: WireProtocol, - effective_model: String, - upstream: LlmJsonStream, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TranslatedStream { - upstream, - transcoder: StreamTranscoder::new(source, target, effective_model), - buffered: VecDeque::new(), - upstream_finished: false, - }) -} - -#[cfg(test)] -#[path = "../tests/unit/component_tests.rs"] -mod tests; diff --git a/crates/switchyard/src/contract.rs b/crates/switchyard/src/contract.rs deleted file mode 100644 index 675234ca1..000000000 --- a/crates/switchyard/src/contract.rs +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! JSON compatibility types for the Switchyard Decision API. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// Routing request schema supported by this plugin. -pub const ROUTING_REQUEST_SCHEMA_VERSION: &str = "switchyard.routing_request.v1"; -/// Routing decision schema supported by this plugin. -pub const ROUTING_DECISION_SCHEMA_VERSION: &str = "switchyard.routing_decision.v1"; - -/// Request-time materialization supplied to Switchyard. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RequestMaterialization { - /// Identity, protocol, summary, and attempt only. - None, - /// Baseline summary without current request material. - SummaryOnly, - /// Latest user prompt. - LatestUserPrompt, - /// Bounded recent message window. - RecentMessageWindow, - /// Relay-normalized request plus its provider body. - AnnotatedRequest, - /// Complete provider request body. - FullBody, -} - -/// Switchyard profile selection. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProfile { - /// Profile ID loaded by Switchyard. - pub profile_id: String, - /// Request materialization mode. - pub request_materialization: RequestMaterialization, -} - -/// Normalized Relay identity. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestIdentity { - /// Stable session identifier. - pub session_id: String, - /// Per-request identifier. - pub request_id: String, - /// Optional turn identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub turn_id: Option, - /// Optional parent scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_scope_id: Option, - /// Optional root scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub root_scope_id: Option, - /// Harness name. - pub harness: String, - /// Request source. - pub source: String, - /// Optional resolved work owner. - #[serde(skip_serializing_if = "Option::is_none")] - pub owner_id: Option, - /// Native, explicit, or synthetic identity quality. - pub quality: String, -} - -/// Inbound protocol context. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestProtocol { - /// Inbound protocol profile. - pub inbound_profile: String, - /// Inbound endpoint. - pub inbound_endpoint: String, - /// Response profile expected by the harness. - pub desired_response_profile: String, -} - -/// Cheap provider-request summary. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct RequestSummary { - /// Client-requested model. - #[serde(skip_serializing_if = "Option::is_none")] - pub client_requested_model: Option, - /// Optional prompt-token estimate. - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_token_estimate: Option, - /// Number of tools in the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_count_in_payload: Option, - /// Whether a system prompt is present. - #[serde(skip_serializing_if = "Option::is_none")] - pub has_system_prompt: Option, -} - -/// Routing-attempt context. Additive fields are ignored by older Switchyard servers. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionAttempt { - /// One-indexed routing attempt. - pub routing_attempt: u32, - /// Maximum Decision API attempts. - pub max_routing_attempts: u32, - /// Previously selected backend. - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_route: Option, - /// Reason another decision is requested. - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_reason: Option, -} - -/// Canonical routing request sent by Relay. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingRequest { - /// Schema identifier. - pub schema_version: String, - /// Switchyard profile selection. - pub decision_profile: DecisionProfile, - /// Normalized identity. - pub identity: RequestIdentity, - /// Inbound protocol. - pub protocol: RequestProtocol, - /// Cheap request summary. - pub request_summary: RequestSummary, - /// Optional request materialization. - #[serde(skip_serializing_if = "Option::is_none")] - pub current_request: Option, - /// Attempt metadata. - pub attempt: DecisionAttempt, -} - -/// Router metadata returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProvider { - /// Router name. - pub name: String, - /// Router version. - pub version: String, -} - -/// Selected target returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingTarget { - /// Tier label. - pub tier: String, - /// Selected model. - pub target_model: String, - /// Backend binding ID. - pub backend_id: String, - /// Target protocol profile. - pub target_protocol_profile: String, - /// Target endpoint. - pub target_endpoint: String, -} - -/// Canonical Switchyard routing decision. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingDecision { - /// Schema identifier. - pub schema_version: String, - /// Decision identifier. - pub decision_id: String, - /// Router metadata. - pub router: DecisionProvider, - /// Selected target. - pub route: RoutingTarget, - /// Explicit counterfactual route supplied by routers that define one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub baseline_route: Option, - /// Optional confidence. - #[serde(default)] - pub confidence: Option, - /// Optional reason code. - #[serde(default)] - pub reason_code: Option, - /// Optional reason summary. - #[serde(default)] - pub reason_summary: Option, - /// Additive router metadata. - #[serde(default)] - pub metadata: BTreeMap, - /// Unknown additive response fields. - #[serde(flatten)] - pub extra: BTreeMap, -} diff --git a/crates/switchyard/src/lib.rs b/crates/switchyard/src/lib.rs deleted file mode 100644 index 238d4337f..000000000 --- a/crates/switchyard/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![deny(rustdoc::broken_intra_doc_links, rustdoc::private_intra_doc_links)] - -//! First-party Switchyard Decision API routing plugin for NeMo Relay. - -pub mod component; -pub mod contract; -mod stream_translation; -mod translation; - -pub use component::{ - ContextMode, ProtocolDefaults, RoutingMode, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, - TargetBinding, WireProtocol, deregister_switchyard_component, register_switchyard_component, - validate_switchyard_atof_configuration, -}; diff --git a/crates/switchyard/src/stream_translation.rs b/crates/switchyard/src/stream_translation.rs deleted file mode 100644 index 808834a2b..000000000 --- a/crates/switchyard/src/stream_translation.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Relay stream adapter for Switchyard's provider translation engine. - -use nemo_relay::error::{FlowError, Result}; -use serde_json::Value as Json; -use switchyard_translation::{StreamTranslationState, TranslationEngine}; - -use crate::component::WireProtocol; -use crate::translation::wire_format; - -pub(crate) struct StreamTranscoder { - engine: TranslationEngine, - source: WireProtocol, - target: WireProtocol, - state: StreamTranslationState, -} - -impl StreamTranscoder { - pub(crate) fn new( - source: WireProtocol, - target: WireProtocol, - effective_model: impl Into, - ) -> Self { - let effective_model = effective_model.into(); - let mut state = StreamTranslationState::new(wire_format(source), wire_format(target)); - state.model = Some(effective_model.clone()); - state.target_model = Some(effective_model); - Self { - engine: TranslationEngine::default(), - source, - target, - state, - } - } - - pub(crate) fn transcode(&mut self, chunk: &Json) -> Result> { - if unsupported_stream_chunk(self.source, chunk) { - return Err(FlowError::InvalidArgument( - "provider-specific streaming extension cannot be translated safely".into(), - )); - } - self.engine - .translate_event( - &mut self.state, - wire_format(self.source), - wire_format(self.target), - chunk, - ) - .map_err(|error| { - FlowError::InvalidArgument(format!("Switchyard stream translation failed: {error}")) - }) - } - - pub(crate) fn finish(&mut self) -> Result> { - self.engine - .finish_stream(&mut self.state, wire_format(self.target)) - .map_err(|error| { - FlowError::InvalidArgument(format!( - "Switchyard stream finalization failed: {error}" - )) - }) - } -} - -fn unsupported_stream_chunk(source: WireProtocol, chunk: &Json) -> bool { - match source { - WireProtocol::OpenaiChat => { - chunk["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["delta"].get("audio").is_some() - || choice["delta"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => { - chunk.get("type").and_then(Json::as_str) == Some("response.output_item.added") - && matches!( - chunk["item"].get("type").and_then(Json::as_str), - Some("reasoning" | "computer_call" | "web_search_call") - ) - } - WireProtocol::AnthropicMessages => { - chunk.get("type").and_then(Json::as_str) == Some("content_block_start") - && !matches!( - chunk["content_block"].get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - } - } -} diff --git a/crates/switchyard/src/translation.rs b/crates/switchyard/src/translation.rs deleted file mode 100644 index d1640226c..000000000 --- a/crates/switchyard/src/translation.rs +++ /dev/null @@ -1,316 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use nemo_relay::api::llm::LlmRequest; -use nemo_relay::error::{FlowError, Result}; -use serde_json::{Map, Value as Json}; -use switchyard_translation::{ - ContentBlock, DeterministicIdPolicy, ImageSource, LlmRequest as TranslationRequest, - LossyConversionPolicy, PreservationPolicy, Role, TargetCapabilities, TranslationDiagnostic, - TranslationEngine, TranslationPolicy, UnknownFieldPolicy, WireFormat, -}; - -use crate::component::WireProtocol; - -pub(crate) fn translation_engine() -> TranslationEngine { - TranslationEngine::default() -} - -pub(crate) fn decode_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &LlmRequest, -) -> Result { - let output = engine - .decode_request( - wire_format(protocol), - &request.content, - &translation_policy(), - ) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(output.request) -} - -pub(crate) fn encode_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &TranslationRequest, - headers: Map, -) -> Result { - let output = engine - .encode_request(wire_format(protocol), request, &translation_policy()) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(LlmRequest { - headers, - content: output.body, - }) -} - -pub(crate) fn validate_portable_request( - engine: &TranslationEngine, - protocol: WireProtocol, - request: &LlmRequest, -) -> Result<()> { - const RESTRICTED_KEYS: &[&str] = &[ - "cache_control", - "audio", - "thinking", - "computer_use", - "server_tool_use", - ]; - if protocol == WireProtocol::AnthropicMessages - && contains_invalid_anthropic_image_source(&request.content) - { - return Err(FlowError::InvalidArgument( - "request uses an unsupported or malformed Anthropic image source".into(), - )); - } - if contains_any_key_recursive(&request.content, RESTRICTED_KEYS) { - return Err(FlowError::InvalidArgument( - "request uses a provider-specific extension that requires same-protocol fail-open" - .into(), - )); - } - let decoded = decode_request(engine, protocol, request)?; - if decoded.reasoning.effort.is_some() - || decoded.reasoning.raw.is_some() - || decoded - .extensions - .fields - .iter() - .any(|(key, value)| key != "stream_options" || !portable_stream_options(value)) - || request_contains_unsupported_content(&decoded) - { - return Err(FlowError::InvalidArgument( - "request uses provider-specific fields that cannot be translated safely".into(), - )); - } - Ok(()) -} - -pub(crate) fn latest_user_prompt(request: &TranslationRequest) -> Option { - request - .messages - .iter() - .rev() - .find(|message| message.role == Role::User) - .and_then(|message| message.text_content("\n")) -} - -pub(crate) fn recent_message_window( - request: &TranslationRequest, - count: usize, -) -> TranslationRequest { - let mut window = request.clone(); - let split = window.messages.len().saturating_sub(count); - window.messages = window.messages.split_off(split); - window -} - -pub(crate) fn translate_response( - engine: &TranslationEngine, - source: WireProtocol, - target: WireProtocol, - response: &Json, -) -> Result { - if source == target { - return Ok(response.clone()); - } - ensure_portable_response(source, response)?; - let output = engine - .translate_response( - wire_format(source), - wire_format(target), - response, - &translation_policy(), - ) - .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; - Ok(output.body) -} - -pub(crate) const fn wire_format(protocol: WireProtocol) -> WireFormat { - match protocol { - WireProtocol::OpenaiChat => WireFormat::OpenAiChat, - WireProtocol::OpenaiResponses => WireFormat::OpenAiResponses, - WireProtocol::AnthropicMessages => WireFormat::AnthropicMessages, - } -} - -fn translation_policy() -> TranslationPolicy { - TranslationPolicy { - unknown_field_policy: UnknownFieldPolicy::Reject, - lossy_conversion_policy: LossyConversionPolicy::Reject, - deterministic_ids: DeterministicIdPolicy::GenerateStable { - prefix: "relay".into(), - }, - preservation: PreservationPolicy::Disabled, - target_capabilities: TargetCapabilities::default(), - } -} - -fn translation_error(error: switchyard_translation::TranslationError) -> FlowError { - FlowError::InvalidArgument(format!("Switchyard translation failed: {error}")) -} - -fn ensure_no_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { - if diagnostics.is_empty() { - Ok(()) - } else { - Err(FlowError::InvalidArgument(format!( - "Switchyard translation was not lossless: {diagnostics:?}" - ))) - } -} - -fn portable_stream_options(value: &Json) -> bool { - let Some(options) = value.as_object() else { - return false; - }; - options.len() == 1 && options.get("include_usage").is_some_and(Json::is_boolean) -} - -fn request_contains_unsupported_content(request: &TranslationRequest) -> bool { - request - .instructions - .iter() - .flat_map(|instruction| instruction.content.iter()) - .chain( - request - .messages - .iter() - .flat_map(|message| message.content.iter()), - ) - .any(unsupported_content_block) -} - -fn unsupported_content_block(block: &ContentBlock) -> bool { - match block { - ContentBlock::Text { .. } | ContentBlock::Refusal { .. } | ContentBlock::ToolCall(_) => { - false - } - ContentBlock::Image { source } => invalid_image_source(source), - ContentBlock::ToolResult(result) => result.content.iter().any(unsupported_content_block), - ContentBlock::Reasoning { .. } - | ContentBlock::Audio { .. } - | ContentBlock::Video { .. } - | ContentBlock::File { .. } - | ContentBlock::Unknown { .. } => true, - } -} - -fn invalid_image_source(source: &ImageSource) -> bool { - match source { - ImageSource::Url { url, .. } => { - url.starts_with("data:") && base64_data_uri_parts(url).is_none() - } - ImageSource::Base64 { media_type, data } => { - media_type.as_deref().is_none_or(str::is_empty) || data.is_empty() - } - ImageSource::Raw(_) => true, - } -} - -fn base64_data_uri_parts(url: &str) -> Option<(&str, &str)> { - let (metadata, data) = url.strip_prefix("data:")?.split_once(',')?; - let media_type = metadata.strip_suffix(";base64")?; - (!media_type.is_empty() && !data.is_empty()).then_some((media_type, data)) -} - -fn contains_invalid_anthropic_image_source(value: &Json) -> bool { - match value { - Json::Object(object) => { - if object.get("type").and_then(Json::as_str) == Some("image") { - let Some(source) = object.get("source").and_then(Json::as_object) else { - return true; - }; - match source.get("type").and_then(Json::as_str) { - Some("url") => source - .get("url") - .and_then(Json::as_str) - .is_none_or(str::is_empty), - Some("base64") => { - source - .get("media_type") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - || source - .get("data") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - } - _ => true, - } - } else { - object.values().any(contains_invalid_anthropic_image_source) - } - } - Json::Array(items) => items.iter().any(contains_invalid_anthropic_image_source), - _ => false, - } -} - -fn contains_any_key_recursive(value: &Json, keys: &[&str]) -> bool { - match value { - Json::Object(object) => { - object.keys().any(|key| keys.contains(&key.as_str())) - || object - .values() - .any(|value| contains_any_key_recursive(value, keys)) - } - Json::Array(items) => items - .iter() - .any(|value| contains_any_key_recursive(value, keys)), - _ => false, - } -} - -fn ensure_portable_response(protocol: WireProtocol, response: &Json) -> Result<()> { - let unsupported = - match protocol { - WireProtocol::OpenaiChat => { - response["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["message"].get("audio").is_some() - || choice["message"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => response["output"] - .as_array() - .into_iter() - .flatten() - .any(|item| { - matches!( - item.get("type").and_then(Json::as_str), - Some( - "reasoning" - | "computer_call" - | "computer_call_output" - | "web_search_call" - ) - ) - }), - WireProtocol::AnthropicMessages => response["content"] - .as_array() - .into_iter() - .flatten() - .any(|block| { - !matches!( - block.get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - }), - }; - if unsupported { - Err(FlowError::InvalidArgument( - "provider-specific response extension cannot be translated safely".into(), - )) - } else { - Ok(()) - } -} diff --git a/crates/switchyard/tests/contract.rs b/crates/switchyard/tests/contract.rs deleted file mode 100644 index 7051f6a56..000000000 --- a/crates/switchyard/tests/contract.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for the public Switchyard Decision API contract. - -use nemo_relay_switchyard::contract::{DecisionAttempt, RoutingDecision}; -use serde_json::json; - -#[test] -fn current_switchyard_decision_contract_accepts_additive_fields() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"}, - "route": { - "tier": "capable", - "target_model": "model-a", - "backend_id": "backend-a", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.8, - "future_field": {"safe": true} - })) - .unwrap(); - assert_eq!( - decision.extra.get("future_field"), - Some(&json!({"safe": true})) - ); - assert!(decision.baseline_route.is_none()); -} - -#[test] -fn current_switchyard_decision_contract_accepts_an_explicit_baseline() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage", "version": "1"}, - "route": { - "tier": "efficient", - "target_model": "model-small", - "backend_id": "backend-small", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "baseline_route": { - "tier": "capable", - "target_model": "model-large", - "backend_id": "backend-large", - "target_protocol_profile": "anthropic_messages", - "target_endpoint": "/v1/messages" - } - })) - .unwrap(); - let baseline = decision.baseline_route.unwrap(); - assert_eq!(baseline.backend_id, "backend-large"); - assert_eq!(baseline.target_protocol_profile, "anthropic_messages"); -} - -#[test] -fn malformed_decisions_are_rejected_by_serde() { - let missing_route = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"} - }); - assert!(serde_json::from_value::(missing_route).is_err()); -} - -#[test] -fn additive_retry_metadata_stays_optional_on_the_wire() { - let attempt: DecisionAttempt = serde_json::from_value(json!({ - "routing_attempt": 1, - "max_routing_attempts": 4 - })) - .unwrap(); - assert!(attempt.previous_route.is_none()); - assert!(attempt.retry_reason.is_none()); -} diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs deleted file mode 100644 index 670d9159b..000000000 --- a/crates/switchyard/tests/unit/component_tests.rs +++ /dev/null @@ -1,1827 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Unit tests for the Switchyard Relay plugin component. - -use std::future::Future; -use std::pin::Pin; -use std::sync::Mutex; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::task::{Context, Poll}; - -use axum::{ - Json as AxumJson, Router, - extract::State, - http::StatusCode, - routing::{get, post}, -}; -use futures_util::{Stream, stream as futures_stream}; -use nemo_relay::api::event::{Event, ScopeCategory}; -use nemo_relay::api::llm::{ - LlmCallExecuteParams, LlmStreamCallExecuteParams, llm_call_execute, llm_stream_call_execute, -}; -use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmStreamExecutionNextFn, LlmStreamInner}; -use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; -use nemo_relay::codec::optimization::LlmOptimizationSummaryStatus; -use nemo_relay::error::{UpstreamFailure, UpstreamFailureClass}; -use nemo_relay::plugin::rollback_registrations; - -use super::*; - -struct CloseTrackingStream { - close_calls: Arc, - closed: bool, -} - -impl Stream for CloseTrackingStream { - type Item = FlowResult; - - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(None) - } -} - -impl LlmStreamInner for CloseTrackingStream { - fn close( - mut self: Pin<&mut Self>, - ) -> Pin> + Send + '_>> { - self.closed = true; - self.close_calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok(()) }) - } -} - -fn close_tracking_stream(close_calls: Arc) -> LlmJsonStream { - LlmJsonStream::from_closeable(CloseTrackingStream { - close_calls, - closed: false, - }) -} - -fn prefixed_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(json!({"first": true}))), - upstream, - }) -} - -fn terminal_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - mark_terminal_stream(upstream, "test", "enforce", json!({"route": "test"})) -} - -fn translated_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - upstream, - ) -} - -fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { - TargetBinding { - model: model.into(), - protocol, - endpoint: protocol.endpoint().into(), - base_url: "http://127.0.0.1:9999".into(), - headers: BTreeMap::new(), - header_env: BTreeMap::new(), - } -} - -fn config(decision_api_url: String) -> SwitchyardConfig { - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets: BTreeMap::from([ - ( - "selected-chat".into(), - binding(WireProtocol::OpenaiChat, "selected"), - ), - ( - "baseline-chat".into(), - binding(WireProtocol::OpenaiChat, "baseline"), - ), - ( - "fallback-chat".into(), - binding(WireProtocol::OpenaiChat, "fallback"), - ), - ( - "fallback-responses".into(), - binding(WireProtocol::OpenaiResponses, "fallback"), - ), - ( - "fallback-anthropic".into(), - binding(WireProtocol::AnthropicMessages, "fallback"), - ), - ]), - default_targets: ProtocolDefaults { - openai_chat: "fallback-chat".into(), - openai_responses: "fallback-responses".into(), - anthropic_messages: "fallback-anthropic".into(), - }, - ..SwitchyardConfig::default() - } -} - -fn decision() -> RoutingDecision { - RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "decision-1".into(), - router: crate::contract::DecisionProvider { - name: "stage_router".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "selected".into(), - backend_id: "selected-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }, - baseline_route: Some(crate::contract::RoutingTarget { - tier: "capable".into(), - target_model: "baseline".into(), - backend_id: "baseline-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }), - confidence: Some(0.9), - reason_code: Some("test".into()), - reason_summary: None, - metadata: BTreeMap::from([ - ("feature_state".into(), json!("fresh")), - ("snapshot_age_millis".into(), json!(37)), - ("snapshot_max_age_millis".into(), json!(300_000)), - ]), - extra: BTreeMap::new(), - } -} - -fn chat_request() -> LlmRequest { - LlmRequest { - headers: Map::new(), - content: json!({ - "model": "inbound", - "messages": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "answer"}, - {"role": "user", "content": "latest"} - ] - }), - } -} - -fn request(protocol: WireProtocol) -> LlmRequest { - let content = match protocol { - WireProtocol::OpenaiChat => chat_request().content, - WireProtocol::OpenaiResponses => { - json!({"model": "inbound", "instructions": "system", "input": "latest"}) - } - WireProtocol::AnthropicMessages => { - json!({"model": "inbound", "system": "system", "max_tokens": 32, "messages": [{"role": "user", "content": "latest"}]}) - } - }; - LlmRequest { - headers: Map::new(), - content, - } -} - -fn chat_response() -> Json { - json!({ - "id": "chat-1", "object": "chat.completion", "model": "selected", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} - }) -} - -fn chat_chunk(text: &str, finish_reason: Json) -> Json { - json!({ - "id": "chat-1", "object": "chat.completion.chunk", "model": "selected", - "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": finish_reason}] - }) -} - -#[test] -fn wire_protocol_and_plugin_lifecycle_contracts_are_stable() { - for (protocol, label, endpoint) in [ - ( - WireProtocol::OpenaiChat, - "openai_chat", - "/v1/chat/completions", - ), - ( - WireProtocol::OpenaiResponses, - "openai_responses", - "/v1/responses", - ), - ( - WireProtocol::AnthropicMessages, - "anthropic_messages", - "/v1/messages", - ), - ] { - assert_eq!(protocol.label(), label); - assert_eq!(protocol.endpoint(), endpoint); - assert_eq!( - WireProtocol::from_call(label, &request(protocol)), - Some(protocol) - ); - assert_eq!( - WireProtocol::from_call("unknown", &request(protocol)), - Some(protocol) - ); - } - assert_eq!( - WireProtocol::from_call( - "unknown", - &LlmRequest { - headers: Map::new(), - content: json!({}) - } - ), - None - ); - assert_eq!(RoutingMode::Enforce.label(), "enforce"); - assert_eq!(RoutingMode::ObserveOnly.label(), "observe_only"); - - let valid = config("http://127.0.0.1:1/v1/routing/decision".into()); - let component: PluginComponentSpec = valid.clone().into(); - assert_eq!(component.kind, SWITCHYARD_PLUGIN_KIND); - assert!(component.enabled); - assert_eq!(component.config["decision_profile_id"], "stage_router"); - - let plugin = SwitchyardPlugin; - assert_eq!(plugin.plugin_kind(), SWITCHYARD_PLUGIN_KIND); - assert!(!plugin.allows_multiple_components()); - assert!(plugin.validate(&component.config).is_empty()); - let diagnostics = plugin.validate(json!({"version": "invalid"}).as_object().unwrap()); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].level, DiagnosticLevel::Error); - assert_eq!(diagnostics[0].code, "switchyard.invalid_config"); -} - -#[tokio::test] -async fn plugin_registration_installs_and_rolls_back_both_execution_intercepts() { - let plugin = SwitchyardPlugin; - let decision_api_url = health_server(StatusCode::OK, r#"{"status":"ok"}"#).await; - for (mode, profile_id) in [ - (RoutingMode::Enforce, "random"), - (RoutingMode::Enforce, "llm"), - (RoutingMode::Enforce, "stage_router"), - (RoutingMode::ObserveOnly, "random"), - (RoutingMode::ObserveOnly, "llm"), - (RoutingMode::ObserveOnly, "stage_router"), - ] { - let mut candidate = config(decision_api_url.clone()); - candidate.mode = mode; - candidate.decision_profile_id = profile_id.into(); - let component: PluginComponentSpec = candidate.into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_retries_transient_sidecar_health_failures() { - let (decision_api_url, attempts) = recovering_health_server(2).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); -} - -#[tokio::test] -async fn plugin_registration_returns_the_final_health_failure_after_retry_exhaustion() { - let (decision_api_url, attempts) = recovering_health_server(usize::MAX).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert!( - error - .to_string() - .contains("returned HTTP 503 Service Unavailable") - ); - assert!(context.into_registrations().is_empty()); -} - -#[tokio::test] -async fn plugin_registration_rejects_unhealthy_or_invalid_sidecars() { - for (status, body, expected) in [ - ( - StatusCode::SERVICE_UNAVAILABLE, - r#"{"status":"starting"}"#, - "returned HTTP 503 Service Unavailable", - ), - (StatusCode::OK, "not-json", "returned invalid JSON"), - ( - StatusCode::OK, - r#"{"status":"starting"}"#, - "did not report status=ok", - ), - ] { - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(health_server(status, body).await).into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!( - error.to_string().contains(expected), - "{error:?} did not contain {expected:?}" - ); - assert!(context.into_registrations().is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_rejects_an_unreachable_sidecar() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = - config(format!("http://{address}/v1/routing/decision")).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!(error.to_string().contains("Switchyard service is required")); - assert!(error.to_string().contains("/health")); - assert!(context.into_registrations().is_empty()); -} - -#[test] -fn configuration_validation_rejects_unsafe_or_ambiguous_bindings() { - let url = "http://127.0.0.1:1/v1/routing/decision".to_string(); - let assert_invalid = |config: SwitchyardConfig, expected: &str| { - let error = SwitchyardRuntime::new(config) - .err() - .expect("config must fail"); - assert!( - error.contains(expected), - "{error:?} did not contain {expected:?}" - ); - }; - - let mut candidate = config(url.clone()); - candidate.version = 2; - assert_invalid(candidate, "unsupported Switchyard config version"); - let mut candidate = config(url.clone()); - candidate.decision_profile_id.clear(); - assert_invalid(candidate, "decision_profile_id"); - let mut candidate = config(url.clone()); - candidate.decision_timeout_millis = 0; - assert_invalid(candidate, "decision_timeout_millis"); - let mut candidate = config(url.clone()); - candidate.max_retries = 11; - assert_invalid(candidate, "max_retries"); - let mut candidate = config(url.clone()); - candidate.recent_message_count = 0; - assert_invalid(candidate, "recent_message_count"); - let mut candidate = config(url.clone()); - candidate.context_mode = ContextMode::AtofRequired; - assert_invalid(candidate, "atof_endpoint_name"); - let mut candidate = config(url.clone()); - candidate.atof_endpoint_name = Some(" switchyard ".into()); - assert_invalid(candidate, "leading or trailing whitespace"); - let candidate = config("file:///tmp/decision".into()); - assert_invalid(candidate, "must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.clear(); - assert_invalid(candidate, "targets must not be empty"); - let mut candidate = config(url.clone()); - candidate.enabled_inbound_profiles.clear(); - assert_invalid(candidate, "enabled_inbound_profiles"); - - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().endpoint = "/wrong".into(); - assert_invalid(candidate, "endpoint must be"); - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().base_url = "file:///tmp/provider".into(); - assert_invalid(candidate, "base_url must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.insert( - "duplicate-chat".into(), - candidate.targets["selected-chat"].clone(), - ); - assert_invalid(candidate, "conflicts with another exact backend binding"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "missing".into(); - assert_invalid(candidate, "default target \"missing\" is not configured"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "fallback-responses".into(); - assert_invalid(candidate, "must use protocol openai_chat"); - - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("bad header".into(), "value".into()); - assert_invalid(candidate, "invalid header name"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("x-test".into(), "bad\nvalue".into()); - assert_invalid(candidate, "invalid header value"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("Authorization".into(), "Bearer static".into()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url.clone()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); - let mut candidate = config(url.clone()); - let target = candidate.targets.get_mut("selected-chat").unwrap(); - target - .headers - .insert("Authorization".into(), "Bearer static".into()); - target.header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url); - candidate - .targets - .get_mut("selected-chat") - .unwrap() - .header_env - .insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); -} - -#[test] -fn disabled_protocol_defaults_are_optional() { - let defaults: ProtocolDefaults = serde_json::from_value(json!({ - "anthropic_messages": "fallback-anthropic" - })) - .expect("omitted disabled protocol defaults should deserialize"); - assert!(defaults.openai_chat.is_empty()); - assert!(defaults.openai_responses.is_empty()); - - let mut candidate = config("http://127.0.0.1:1/v1/routing/decision".into()); - candidate.enabled_inbound_profiles = BTreeSet::from([WireProtocol::AnthropicMessages]); - candidate.default_targets.openai_chat.clear(); - candidate.default_targets.openai_responses.clear(); - - SwitchyardRuntime::new(candidate).expect("Anthropic-only configuration should validate"); -} - -#[test] -fn atof_cross_component_validation_rejects_invalid_endpoint_names_before_lookup() { - for (name, expected) in [ - (" ", "atof_endpoint_name must be non-empty when configured"), - ( - " switchyard ", - "atof_endpoint_name must not have leading or trailing whitespace", - ), - ] { - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some(name.into()); - let plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - - assert_eq!( - validate_switchyard_atof_configuration(&plugin_config).unwrap_err(), - expected - ); - } -} - -#[test] -fn atof_cross_component_validation_reports_each_activation_mismatch() { - assert!(validate_switchyard_atof_configuration(&PluginConfig::default()).is_ok()); - - let payload = PluginConfig { - components: vec![config("http://switchyard.test/v1/routing/decision".into()).into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&payload).is_ok()); - - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": {"enabled": true, "sinks": [{ - "type": "stream", - "name": "other", - "url": "http://events.test/v1/atof/events", - "header_env": {"authorization": "TOKEN"} - }]}}) - .as_object() - .unwrap() - .clone(), - }); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("requires named ATOF endpoint") - ); - - plugin_config.components[1].config["atof"]["sinks"][0]["name"] = json!("switchyard"); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("websocket"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("transport = http_post") - ); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("http_post"); - let duplicate = plugin_config.components[1].config["atof"]["sinks"][0].clone(); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .push(duplicate); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("exactly one endpoint") - ); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .pop(); - plugin_config.components[1].config["atof"]["sinks"][0]["field_name_policy"] = - json!("snake_case"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("field_name_policy = preserve") - ); - let endpoint = &mut plugin_config.components[1].config["atof"]["sinks"][0]; - endpoint["field_name_policy"] = json!("preserve"); - endpoint.as_object_mut().unwrap().remove("header_env"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("environment-referenced header") - ); -} - -#[tokio::test] -async fn execution_bypasses_inapplicable_calls_and_fails_open_on_extensions() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let passthrough: LlmExecutionNextFn = - Arc::new(|request| Box::pin(async move { Ok(request.content) })); - let unknown = LlmRequest { - headers: Map::new(), - content: json!({"opaque": true}), - }; - assert_eq!( - runtime - .execute_buffered("custom.provider", unknown, Arc::clone(&passthrough)) - .await - .unwrap()["opaque"], - true - ); - - let mut disabled_config = config("http://127.0.0.1:1/v1/routing/decision".into()); - disabled_config - .enabled_inbound_profiles - .remove(&WireProtocol::OpenaiChat); - let disabled = SwitchyardRuntime::new(disabled_config).unwrap(); - assert_eq!( - disabled - .execute_buffered( - "openai.chat_completions", - chat_request(), - Arc::clone(&passthrough), - ) - .await - .unwrap()["model"], - "inbound" - ); - - let mut unsupported = chat_request(); - unsupported.content["thinking"] = json!({"type": "enabled"}); - let response = runtime - .execute_buffered( - "openai.chat_completions", - unsupported, - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); -} - -#[tokio::test] -async fn stream_setup_retries_and_empty_streams_have_one_bounded_fallback() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connect".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })); - } - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); - - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt < 4 { - Vec::new() - } else { - assert_eq!(request.content["model"], "fallback"); - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - assert_eq!(decisions.lock().unwrap().len(), 4); -} - -#[tokio::test] -async fn fallback_setup_failures_preserve_the_provider_error() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let buffered: LlmExecutionNextFn = Arc::new(|_| { - Box::pin(async { Err(FlowError::Internal("buffered fallback failed".into())) }) - }); - let error = runtime - .dispatch_fallback_buffered(WireProtocol::OpenaiChat, chat_request(), buffered, "test") - .await - .unwrap_err(); - assert!(error.to_string().contains("buffered fallback failed")); - - let streaming: LlmStreamExecutionNextFn = - Arc::new(|_| Box::pin(async { Err(FlowError::Internal("stream fallback failed".into())) })); - let error = runtime - .dispatch_fallback_stream(WireProtocol::OpenaiChat, chat_request(), streaming, "test") - .await - .err() - .expect("stream fallback setup must fail"); - assert!(error.to_string().contains("stream fallback failed")); -} - -#[tokio::test] -async fn translated_stream_preserves_success_and_propagates_both_error_sources() { - let source = LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("hello", Json::Null)), - Ok(chat_chunk("", json!("stop"))), - ])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output.iter().all(Result::is_ok)); - assert!(output.iter().any(|item| { - item.as_ref() - .is_ok_and(|chunk| chunk.to_string().contains("hello")) - })); - - let upstream_error = FlowError::Internal("upstream stream failed".into()); - let source = LlmJsonStream::new(futures_stream::iter(vec![Err(upstream_error)])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output[0].is_err()); - - let malformed = LlmJsonStream::new(futures_stream::iter(vec![Ok(json!({ - "choices": [{"delta": {"reasoning_content": "private"}}] - }))])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - malformed, - ) - .collect::>() - .await; - assert!(output[0].is_err()); -} - -#[tokio::test] -async fn stream_adapters_forward_explicit_close_to_the_upstream_stream() { - for make_adapter in [ - prefixed_adapter as fn(LlmJsonStream) -> LlmJsonStream, - terminal_adapter, - translated_adapter, - ] { - let close_calls = Arc::new(AtomicUsize::new(0)); - let mut stream = make_adapter(close_tracking_stream(Arc::clone(&close_calls))); - - stream.close().await.unwrap(); - - assert_eq!(close_calls.load(Ordering::SeqCst), 1); - assert!(stream.next().await.is_none()); - } -} - -#[test] -fn provider_failure_reporting_covers_every_retry_class() { - for (class, label, retryable) in [ - (UpstreamFailureClass::Connection, "connection", true), - (UpstreamFailureClass::Timeout, "timeout", true), - ( - UpstreamFailureClass::RetryableStatus, - "retryable_status", - true, - ), - (UpstreamFailureClass::ContextWindow, "context_window", true), - ( - UpstreamFailureClass::ModelUnavailable, - "model_unavailable", - true, - ), - ( - UpstreamFailureClass::Authentication, - "authentication", - false, - ), - ( - UpstreamFailureClass::InvalidRequest, - "invalid_request", - false, - ), - (UpstreamFailureClass::Other, "other", false), - ] { - let error = FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "failure".into(), - headers: BTreeMap::new(), - class, - }); - assert_eq!(provider_error_class(&error), label); - assert_eq!(error_is_retryable(&error), retryable); - assert_eq!(provider_error_summary(&error), format!("{label}:http_503")); - } - let relay = FlowError::Internal("failure".into()); - assert_eq!(provider_error_class(&relay), "relay"); - assert_eq!(provider_error_summary(&relay), relay.to_string()); -} - -#[test] -fn all_materialization_modes_are_bounded_and_provider_valid() { - for protocol in [ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ] { - for mode in [ - RequestMaterialization::None, - RequestMaterialization::SummaryOnly, - RequestMaterialization::LatestUserPrompt, - RequestMaterialization::RecentMessageWindow, - RequestMaterialization::AnnotatedRequest, - RequestMaterialization::FullBody, - ] { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - config.request_materialization = mode; - config.recent_message_count = 2; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let routing = runtime - .routing_request(protocol, &request(protocol), 1, None) - .unwrap(); - match mode { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => { - assert!(routing.current_request.is_none()) - } - RequestMaterialization::LatestUserPrompt => { - let current = routing.current_request.unwrap(); - assert_eq!(current["latest_user_prompt"], "latest"); - let body = current["body"].clone(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: body, - }, - ) - .unwrap(); - } - RequestMaterialization::RecentMessageWindow => { - let current = routing.current_request.unwrap(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: current["body"].clone(), - }, - ) - .unwrap(); - } - RequestMaterialization::AnnotatedRequest | RequestMaterialization::FullBody => { - assert!(routing.current_request.is_some()) - } - } - } - } -} - -#[test] -fn identity_policy_requires_stable_request_scope_only_for_atof_profiles() { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - let payload_runtime = SwitchyardRuntime::new(config.clone()).unwrap(); - let synthetic = payload_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - assert_eq!(synthetic.identity.quality, "synthetic"); - - config.context_mode = ContextMode::AtofRequired; - config.atof_endpoint_name = Some("switchyard".into()); - let atof_runtime = SwitchyardRuntime::new(config).unwrap(); - assert!( - atof_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .is_err() - ); - let mut stable = chat_request(); - stable - .headers - .insert("x-nemo-relay-session-id".into(), json!("session-1")); - stable - .headers - .insert("x-nemo-relay-request-id".into(), json!("request-1")); - let routed = atof_runtime - .routing_request(WireProtocol::OpenaiChat, &stable, 1, None) - .unwrap(); - assert_eq!(routed.identity.quality, "explicit"); -} - -#[test] -fn exact_target_validation_rejects_any_switchyard_drift() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - assert!(runtime.validate_decision(&decision()).is_ok()); - let mut drifted = decision(); - drifted.route.target_model = "unbound-model".into(); - assert!(runtime.validate_decision(&drifted).is_err()); - drifted = decision(); - drifted.route.backend_id = "unknown".into(); - assert!(runtime.validate_decision(&drifted).is_err()); -} - -#[test] -fn routing_contribution_requires_an_exact_independent_baseline_binding() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - - let contribution = runtime.routing_contribution(&decision(), 2, true).unwrap(); - assert!(contribution.applied); - assert_eq!( - contribution.kind.as_str(), - LlmOptimizationKind::MODEL_ROUTING - ); - let transition = contribution.model_transition.unwrap(); - assert_eq!(transition.baseline.unwrap().model, "baseline"); - assert_eq!(transition.effective.unwrap().model, "selected"); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["feature_state"], - "fresh" - ); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["snapshot_age_millis"], - 37 - ); - assert_eq!( - contribution.payload_schema.as_ref().unwrap().name, - ROUTING_CONTRIBUTION_SCHEMA - ); - - let observed = runtime.routing_contribution(&decision(), 1, false).unwrap(); - assert!(!observed.applied); - - let mut missing = decision(); - missing.baseline_route = None; - assert!(runtime.routing_contribution(&missing, 1, true).is_none()); - - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - assert!(runtime.routing_contribution(&drifted, 1, true).is_none()); - assert!(runtime.validate_decision(&drifted).is_ok()); -} - -#[test] -fn routing_decision_mark_has_canonical_shape_and_mirrored_identity() { - let subscriber_name = format!("switchyard-mark-shape-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let routing_request = runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - runtime.emit_decision(&routing_request, &decision(), 1, false, 17); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - - let event = events - .lock() - .unwrap() - .iter() - .map(Event::to_json_value) - .find(|event| { - event["name"] == "switchyard.routing.decision" - && event["metadata"]["session_id"] == routing_request.identity.session_id - && event["metadata"]["request_id"] == routing_request.identity.request_id - }) - .expect("decision mark should be captured"); - assert_eq!(event["kind"], "mark"); - assert_eq!(event["category"], "custom"); - assert_eq!( - event["category_profile"]["subtype"], - "switchyard.routing.decision" - ); - assert_eq!(event["data_schema"]["name"], ROUTING_MARK_SCHEMA); - assert_eq!(event["data_schema"]["version"], "1"); - assert_eq!(event["data"]["profile_id"], "stage_router"); - assert_eq!(event["data"]["selected_model"], "selected"); - assert_eq!(event["data"]["latency_ms"], 17); - assert_eq!(event["data"]["router_metadata"]["feature_state"], "fresh"); - assert_eq!(event["data"]["router_metadata"]["snapshot_age_millis"], 37); - assert_eq!( - event["metadata"]["session_id"], - routing_request.identity.session_id - ); - assert_eq!( - event["metadata"]["request_id"], - routing_request.identity.request_id - ); -} - -#[test] -fn atof_required_cross_component_validation_is_context_sensitive() { - let mut switchyard = config("http://switchyard.test:8080/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&plugin_config).is_err()); - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": { - "enabled": true, - "sinks": [{ - "type": "stream", - "name": "switchyard", - "url": "http://switchyard.test:8080/v1/atof/events", - "transport": "http_post", - "field_name_policy": "preserve", - "header_env": {"authorization": "SWITCHYARD_TOKEN"} - }] - }}) - .as_object() - .unwrap() - .clone(), - }); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); -} - -#[derive(Clone)] -struct DecisionState { - requests: Arc>>, - decision: RoutingDecision, -} - -async fn decision_handler( - State(state): State, - AxumJson(request): AxumJson, -) -> AxumJson { - state.requests.lock().unwrap().push(request); - AxumJson(state.decision) -} - -async fn decision_server() -> (String, Arc>>) { - decision_server_for(decision()).await -} - -async fn decision_server_for( - decision: RoutingDecision, -) -> (String, Arc>>) { - let requests = Arc::new(Mutex::new(Vec::new())); - let state = DecisionState { - requests: Arc::clone(&requests), - decision, - }; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new() - .route("/v1/routing/decision", post(decision_handler)) - .with_state(state), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), requests) -} - -async fn health_server(status: StatusCode, body: &'static str) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || async move { (status, [("content-type", "application/json")], body) }), - ), - ) - .await - .unwrap(); - }); - format!("http://{address}/nested/v1/routing/decision?source=test#ignored") -} - -async fn recovering_health_server(failures_before_ready: usize) -> (String, Arc) { - let attempts = Arc::new(AtomicUsize::new(0)); - let captured = Arc::clone(&attempts); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || { - let attempts = Arc::clone(&captured); - async move { - if attempts.fetch_add(1, Ordering::SeqCst) < failures_before_ready { - ( - StatusCode::SERVICE_UNAVAILABLE, - AxumJson(json!({"status": "starting"})), - ) - } else { - (StatusCode::OK, AxumJson(json!({"status": "ok"}))) - } - } - }), - ), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), attempts) -} - -async fn managed_buffered_events( - runtime: SwitchyardRuntime, - next: LlmExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_buffered("openai.chat_completions", request, next) - .await - }) - }); - llm_call_execute( - LlmCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .build(), - ) - .await - .unwrap(); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} - -async fn managed_stream_events( - runtime: SwitchyardRuntime, - next: LlmStreamExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-stream-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmStreamExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_stream("openai.chat_completions", request, next) - .await - }) - }); - let mut stream = llm_stream_call_execute( - LlmStreamCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .collector(Box::new(|_| Ok(()))) - .finalizer(Box::new(|| json!({"done": true}))) - .build(), - ) - .await - .unwrap(); - while stream.next().await.is_some() {} - drop(stream); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} - -#[tokio::test] -async fn buffered_accounting_records_only_the_terminal_committed_route() { - let (url, _) = decision_server().await; - let successful = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| Box::pin(async { Ok(chat_response()) })), - ) - .await; - let marks = successful - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].data().unwrap()["applied"], true); - assert_eq!( - marks[0].data().unwrap()["model_transition"]["baseline"]["model"], - "baseline" - ); - let summary = successful - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn retry_then_success_records_one_terminal_route_with_matching_mark_identity() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry once".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert_eq!(requests[1].attempt.routing_attempt, 2); - drop(requests); - - let start = events - .iter() - .find(|event| { - event.name() == "openai.chat_completions" - && event.scope_category() == Some(ScopeCategory::Start) - }) - .unwrap(); - let marks = events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].parent_uuid(), Some(start.uuid())); - assert_eq!(marks[0].data().unwrap()["payload"]["routing_attempt"], 2); - - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - let contribution = &summary.contributions[0]; - assert!(contribution.applied); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!(marks[0].data().unwrap()["id"], json!(contribution.id)); - assert_eq!( - marks[0].data().unwrap()["sequence"], - json!(contribution.sequence) - ); -} - -#[tokio::test] -async fn oversized_decision_metadata_limits_accounting_without_failing_provider_success() { - let mut oversized = decision(); - oversized - .metadata - .insert("oversized_evidence".into(), json!("x".repeat(20_000))); - let (url, _) = decision_server_for(oversized).await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(chat_response()) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.status, LlmOptimizationSummaryStatus::Partial); - assert!(summary.contributions.is_empty()); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "contribution_limit_exceeded") - ); -} - -#[tokio::test] -async fn malformed_baseline_is_telemetry_only_and_does_not_change_the_selected_route() { - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - let (url, _) = decision_server_for(drifted).await; - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "selected"); - Ok(chat_response()) - }) - }), - ) - .await; - assert!(events.iter().any(|event| { - event.name() == "switchyard.routing.error" - && event.data().is_some_and(|data| { - data["error_class"] == "baseline_binding" - && data["error"] - .as_str() - .is_some_and(|error| error.contains("exact Relay binding")) - }) - })); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn observe_only_accounting_is_visible_but_not_applied() { - let (url, _) = decision_server().await; - let mut observe = config(url); - observe.mode = RoutingMode::ObserveOnly; - let events = managed_buffered_events( - SwitchyardRuntime::new(observe).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await; - let contribution = events - .iter() - .find(|event| event.name() == "nemo_relay.llm.optimization") - .and_then(Event::data) - .unwrap(); - assert_eq!(contribution["applied"], false); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(!summary.contributions[0].applied); - assert!(summary.tokens_saved.total_tokens.is_none()); -} - -#[tokio::test] -async fn streaming_accounting_commits_on_the_first_successful_item_only() { - let (url, _) = decision_server().await; - let committed = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| { - Box::pin(async { - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }), - ) - .await; - assert_eq!( - committed - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let items = if seen.fetch_add(1, Ordering::SeqCst) == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - }))] - } else { - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn committed_stream_error_keeps_one_route_and_never_redispatches() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed after commit".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]))) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); - assert_eq!( - events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "stream_interrupted") - ); -} - -#[tokio::test] -async fn retry_exhaustion_redecides_four_times_then_dispatches_fallback_once() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst) + 1; - if attempt <= 4 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "temporarily unavailable".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })); - } - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }); - let response = runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 4); - assert_eq!(requests[3].attempt.routing_attempt, 4); - assert_eq!( - requests[3].attempt.previous_route.as_deref(), - Some("selected-chat") - ); -} - -#[tokio::test] -async fn non_retryable_provider_failure_bypasses_retry_loop() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - if attempt == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })); - } - Ok(chat_response()) - }) - }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -#[tokio::test] -async fn observe_only_records_one_decision_and_dispatches_only_the_trusted_default() { - let (url, decisions) = decision_server().await; - let mut config = config(url); - config.mode = RoutingMode::ObserveOnly; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -#[tokio::test] -async fn streaming_retries_before_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - }))] - } else { - vec![Ok(chat_chunk("ok", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let stream = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 1); - assert!(output[0].is_ok()); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); -} - -#[tokio::test] -async fn streaming_never_retries_after_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - let items = vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let stream = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 2); - assert!(output[0].is_ok()); - assert!(output[1].is_err()); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -fn responses_config(decision_api_url: String) -> SwitchyardConfig { - let targets = BTreeMap::from([ - ( - "resp-a".into(), - binding(WireProtocol::OpenaiResponses, "model-a"), - ), - ( - "resp-b".into(), - binding(WireProtocol::OpenaiResponses, "model-b"), - ), - ]); - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets, - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: "resp-a".into(), - anthropic_messages: String::new(), - }, - enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiResponses]), - ..SwitchyardConfig::default() - } -} - -fn responses_decision() -> RoutingDecision { - RoutingDecision { - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "model-b".into(), - backend_id: "resp-b".into(), - target_protocol_profile: "openai_responses".into(), - target_endpoint: "/v1/responses".into(), - }, - baseline_route: None, - ..decision() - } -} - -// When every configured target shares the inbound protocol, no cross-protocol translation can -// occur, so the portability guard is skipped: a non-portable Responses request (real Codex -// extensions) is routed through the Decision API with its provider-specific fields preserved. -#[tokio::test] -async fn same_protocol_targets_route_nonportable_streaming_requests() { - let (url, decisions) = decision_server_for(responses_decision()).await; - let runtime = SwitchyardRuntime::new(responses_config(url)).unwrap(); - let mut nonportable = request(WireProtocol::OpenaiResponses); - nonportable.content["reasoning"] = json!({"effort": "high"}); - nonportable.content["store"] = json!(true); - let next: LlmStreamExecutionNextFn = Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "model-b"); - assert_eq!(request.content["store"], true); - assert_eq!(request.content["reasoning"]["effort"], "high"); - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - json!({"type": "response.output_text.delta", "delta": "ok"}), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.responses", nonportable, next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} diff --git a/docs/about-nemo-relay/release-notes/index.mdx b/docs/about-nemo-relay/release-notes/index.mdx index d36bc2afc..b35f9d255 100644 --- a/docs/about-nemo-relay/release-notes/index.mdx +++ b/docs/about-nemo-relay/release-notes/index.mdx @@ -42,6 +42,8 @@ compatibility information applies to the current 0.7 prerelease. - Native and worker plugins can resolve invocation-scoped codec capabilities. Request sanitizers can decode and encode requests, and response sanitizers can decode responses. +- Switchyard is distributed as an external native dynamic plugin. The built-in + service-backed crate and CLI feature are no longer included. - The Observability plugin now uses one typed, multi-endpoint OpenTelemetry section for `full`, `gen_ai`, and `openinference` trace projections. - The `gen_ai` projection implements the pinned @@ -117,9 +119,6 @@ their values cannot be isolated between endpoints. - Codex 0.143 does not expose `SessionEnd`, and Codex multi-agent v2 encrypts delegated-task payloads that Relay cannot decrypt or reliably link. - The Node.js binding and package workflows require Node.js 24 or later. -- Switchyard is experimental, excluded from default CLI builds, and requires a - separately managed compatible service and a named authenticated ATOF HTTP - stream sink. - OpenClaw has public hook-backed telemetry. Its security and optimization coverage is partial because it does not own a managed execution path. - The NeMo Guardrails remote backend inherits its configured service's diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index b96315a2f..040356018 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -33,9 +33,8 @@ entries in `plugins.toml`. Guardrails-backed policy checks. - [PII Redaction](/configure-plugins/pii-redaction/about) sanitizes sensitive data in observability payloads. -- [Switchyard (Experimental)](/configure-plugins/switchyard/about) routes - requests using decisions from a separately running Switchyard Decision API - service. +- [Switchyard](/configure-plugins/switchyard/about) documents migration to + Switchyard's external native dynamic plugin. - [Model Pricing](/configure-plugins/model-pricing) configures catalog sources for cost estimates on managed LLM responses. diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx index 321a4d43d..f0d859ae9 100644 --- a/docs/configure-plugins/switchyard/about.mdx +++ b/docs/configure-plugins/switchyard/about.mdx @@ -1,204 +1,94 @@ --- -title: "Switchyard (Experimental)" -sidebar-title: "Switchyard (Experimental)" -description: "Set up and validate the experimental Switchyard Decision API integration for NeMo Relay 0.6.0." +title: "Switchyard External Plugin" +sidebar-title: "Switchyard" +description: "Migrate from Relay's built-in Switchyard service integration to Switchyard's native dynamic plugin." position: 5 --- -import { MermaidStyles } from "@/components/MermaidStyles"; - {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -> **Experimental:** The Switchyard integration is an early-access feature. It -> is not enabled in default Relay builds, its configuration and contracts can -> change, and the current deployment requires a separately running Switchyard -> Decision API service. - -> **NeMo Relay 0.6.0 architecture:** This release uses the external Switchyard -> Decision API from the -> [`topic/nemo-relay-integration`](https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration) -> branch. - -Switchyard is an LLM routing decision engine. The `nemo-relay-switchyard` -plugin asks Switchyard which configured target should handle an LLM request. -Relay validates the decision and performs the authenticated provider request. -For example, a routing profile can send a simple prompt to a lower-cost model -and a complex prompt to a more capable model. - -## Architecture - -The following diagram shows the NeMo Relay 0.6.0 service boundary: - - - -```mermaid -flowchart LR - Agent["Agent"] -->|LLM request| Relay["NeMo Relay"] - Relay -->|routing request| Decision["Switchyard Decision API"] - Decision -->|backend decision| Relay - Relay -->|authenticated provider request| Backend["Selected LLM backend"] - Relay -->|ATOF events| History["Switchyard ATOF accumulator"] - History -. routing history .-> Decision - - class Agent grey-lightest; - class Relay green-lightest; - class Decision yellow-lightest; - class History yellow-lightest; - class Backend teal-lightest; -``` +Switchyard is distributed as a native dynamic plugin rather than a built-in +NeMo Relay component. The external plugin runs Switchyard algorithms in the +gateway process through `libsy`. Relay still owns authenticated provider +dispatch, retries, fallback, and observability. -The components divide responsibility as follows: +This replaces the service-backed `nemo-relay-switchyard` crate, the CLI +`switchyard` build feature, and `[[components]]` entries whose +`kind = "switchyard"`. Relay no longer starts, probes, or calls a Switchyard +Decision API service. -| Component | Responsibilities | -| --- | --- | -| Switchyard | Selects a backend and accumulates ATOF routing history for history-based profiles. | -| NeMo Relay | Owns provider credentials, target bindings, decision validation, protocol translation, dispatch, retries, trusted fallback, and observability. | +## Requirements -Relay does not start or supervise the Switchyard service. Start Switchyard -before Relay activates the plugin. Relay derives the service's root `/health` -URL from `decision_api_url` and fails activation unless the endpoint returns -`{"status":"ok"}`. +- A NeMo Relay build that supports native dynamic plugin API v2. +- A `switchyard-nemo-relay-plugin` binary bundle from a trusted + [Switchyard release](https://github.com/NVIDIA-NeMo/Switchyard/releases). +- A bundle built for the host operating system and architecture. -## Prerequisites +The bundle contains the shared library, `relay-plugin.toml`, configuration +schema, license notices, and integrity metadata. The manifest declares +`compat.native_api = "2"` and plugin ID `nvidia.switchyard`. -Install the following prerequisites before you start: + +Native plugins run inside the Relay gateway process. Verify the bundle's +digest and signature, and install it only from a trusted release. + -| Requirement | Version or Value | -| --- | --- | -| NeMo Relay | Tag `0.6.0` | -| Rust | `1.96.1` | -| Switchyard branch | `topic/nemo-relay-integration` | -| Switchyard commit | `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8` | -| Local commands | `git`, `python3`, and `curl` | +## Install the External Plugin -Keep the repositories in the following sibling layout. The validation script -uses this layout by default: +Unpack the bundle and validate its manifest before registering it: -```text -/ -├── NeMo-Relay/ -└── Switchyard-topic-nemo-relay-integration/ +```bash +nemo-relay plugins validate ./switchyard/relay-plugin.toml +nemo-relay plugins add --project ./switchyard/relay-plugin.toml +nemo-relay plugins inspect nvidia.switchyard +nemo-relay plugins enable nvidia.switchyard +nemo-relay plugins validate nvidia.switchyard ``` -## Build and Run the Compatibility Test +Configure the plugin under its dynamic-plugin entry: -Use the following procedure to clone and pin both repositories: +```toml +[[plugins.dynamic]] +manifest = "./switchyard/relay-plugin.toml" -```bash -mkdir relay-switchyard-0.6 -cd relay-switchyard-0.6 - -git clone --branch 0.6.0 \ - https://github.com/NVIDIA/NeMo-Relay.git \ - NeMo-Relay +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 3 -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - Switchyard-topic-nemo-relay-integration +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 -git -C Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 +[plugins.dynamic.config.default_targets] +openai_chat = "chat-default" -cd NeMo-Relay +[plugins.dynamic.config.targets.chat-default] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 ``` -The Switchyard plugin is excluded from default Relay CLI builds. Build the CLI -with the optional feature explicitly: +Use the configuration schema and examples shipped in the Switchyard bundle as +the source of truth for supported algorithms and fields. -```bash -cargo build -p nemo-relay-cli --features switchyard -``` +## Configuration Migration -Run the real-service compatibility test from the NeMo Relay repository root: +Do not copy the old service component into the dynamic entry. Remove these +service-era fields and dependencies: -```bash -examples/switchyard/run-real-e2e.sh -``` +- `decision_api_url`, decision headers, and health-check configuration. +- `decision_profile_id`, request materialization, and Decision API timeouts. +- Switchyard-specific ATOF ingestion sinks used only by the service. +- The CLI `switchyard` Cargo feature and the `nemo-relay-switchyard` crate. -The script prints the verified Switchyard revision and ends with the following -line when routing succeeds: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` +Move Relay-owned target bindings, trusted defaults, retry limits, and supported +algorithm settings into `[plugins.dynamic.config]`. The external plugin uses +`libsy::Algorithm::run_stream`; it asks Relay to dispatch every selected LLM +call through native API v2 and receives the real response stream or structured +provider failure in return. -The test starts these local processes: - -| Process | Port | Readiness Check or Purpose | -| --- | --- | --- | -| Switchyard Decision API | `4000` | Serves `/health`, routing decisions, and ATOF ingestion. | -| NeMo Relay | `4041` | Serves `/healthz` and the OpenAI-compatible gateway. | -| Deterministic provider | `4101` | Records the model selected for each provider request. | - -The first request has no accumulated ATOF history and routes to -`provider/weak`. The script then sends session and tool events to Relay. Relay -exports those events to Switchyard's ATOF accumulator, so the next buffered -request and the final streaming request route to `provider/strong`. The test -also validates the streamed response before reporting success. - -## Configuration Walkthrough - -The compatibility test uses -[`real-e2e-plugins.toml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-plugins.toml) -and -[`real-e2e-profiles.yaml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-profiles.yaml). -The following settings establish the routing boundary: - -| Setting | Purpose | -| --- | --- | -| `decision_api_url` | Points Relay to the separately running Switchyard Decision API. | -| `decision_profile_id` | Selects the Switchyard routing profile. The profile maps its semantic targets to backend IDs. | -| `targets.` | Binds each returned backend ID to a Relay-owned model, protocol, endpoint, base URL, and credentials. | -| `mode = "enforce"` | Applies a valid Switchyard decision and dispatches to the selected target. | -| `mode = "observe_only"` | Records the hypothetical decision but dispatches once to the trusted same-protocol default. | -| `default_targets` | Defines trusted same-protocol fallbacks for unavailable or invalid decisions and provider failures. | -| `context_mode = "atof_required"` | Requires stable identity and accumulated ATOF history for the selected profile. | -| `atof_endpoint_name` | Selects exactly one named Relay ATOF HTTP stream sink that sends events to Switchyard. | - -For history-based routing, a local ATOF JSONL file is not sufficient. -Switchyard must receive events through the named HTTP sink at -`/v1/atof/events` so its accumulator can provide the history used by the -routing profile. Relay rejects a missing, duplicate, disabled, or invalid -named sink during startup validation. - -For the complete option reference, refer to -[Switchyard Configuration](./configuration.mdx). - -## Troubleshooting - -Use the following table to diagnose common compatibility-test failures: - -| Symptom | Cause and Resolution | -| --- | --- | -| `Switchyard worktree not found: ...` | Place the Switchyard checkout next to `NeMo-Relay` as `Switchyard-topic-nemo-relay-integration`, or set `SWITCHYARD_ROOT` to its absolute path. | -| `Switchyard checkout mismatch: expected ..., found ...` | Check out commit `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8`. Override `SWITCHYARD_EXPECTED_COMMIT` only when deliberately testing a different contract. | -| A build uses Rust `1.96.0` or reports an unsupported toolchain | Run `rustc --version`, install Rust `1.96.1`, and allow the repository's `rust-toolchain.toml` to select it. | -| `timed out waiting for http://127.0.0.1:4000/health` | Review `switchyard.log`. Confirm the pinned checkout builds and port `4000` is free. | -| `timed out waiting for http://127.0.0.1:4041/healthz` or `process ... exited before http://127.0.0.1:4041/healthz became ready` | Review `relay.log`. Confirm the CLI was built with `--features switchyard`, the Switchyard health check passed, and port `4041` is free. | -| The provider cannot start or requests fail | Review `upstream.log` and confirm port `4101` is free. | -| `E2E logs preserved in ...` | The script preserves its temporary directory after a runtime failure and prints the exact path. Inspect `switchyard.log`, `relay.log`, `upstream.log`, and the captured request outputs there. Successful runs remove the directory. | - -## Capabilities - -The integration provides the following capabilities: - -- `enforce` and `observe_only` routing modes. -- OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages inbound - profiles. -- Buffered and streaming protocol translation through Switchyard's - `switchyard-translation` library. -- Exact Relay-owned backend bindings and per-protocol trusted fallbacks. -- Bounded retries before the first streaming item. -- Canonical routing marks and shared LLM optimization accounting. - -## Pages - -For more information, refer to the following pages: - -- [Switchyard Configuration](./configuration.mdx) - provides the complete option and deployment reference. Review its - [experimental limitations](./configuration.mdx#experimental-limitations) - before adopting the integration. -- [Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard) - contains the versioned configuration and validation scripts used by this - guide. +See [Configure Discoverable Plugins](/configure-plugins/discoverable-plugins) +for lifecycle, trust, and policy details. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx deleted file mode 100644 index 2b09b555e..000000000 --- a/docs/configure-plugins/switchyard/configuration.mdx +++ /dev/null @@ -1,358 +0,0 @@ ---- -title: "Switchyard Configuration" -sidebar-title: "Configuration" -description: "Configure the experimental Relay-native Switchyard Decision API plugin." -position: 2 ---- -{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 */} - -> **Experimental NeMo Relay 0.6.0 integration:** This release calls a separately -> running Switchyard Decision API. - -Start with the -[Switchyard 0.6.0 setup and validation guide](./about.mdx) -before using this page as the complete configuration reference. - -The Switchyard component connects Relay's CLI gateway to a separately running -[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API. Relay -owns provider credentials, backend endpoints, dispatch, retries, and trusted -fallbacks. The Switchyard service selects a backend. Relay validates the -decision, translates requests and responses in process through the -`switchyard-translation` library, and dispatches directly to the selected -Relay-owned target. - -## How the Configuration Fits Together - -Configure the integration in four core steps. Profiles that use ATOF history -add the fifth requirement: - -1. Set `decision_api_url` and authentication so Relay can call the Switchyard - Decision API. -2. Select the Switchyard routing profile with `decision_profile_id`. -3. Add a Relay-owned `targets` binding for every Switchyard `backend_id` that - the profile can return. Relay validates the returned backend ID, model, - protocol, and endpoint exactly before dispatch. -4. Choose a trusted same-protocol target for each inbound protocol under - `default_targets`. Relay uses these defaults for observe-only traffic and - fail-open dispatch. -5. Configure profiles that depend on ATOF history, including StageRouter, with - `context_mode = "atof_required"` and `atof_endpoint_name`, then configure the - matching observability stream sink. Relay fails startup validation if the - named sink is missing or invalid. - -The following is a complete component configuration. The referenced -environment variables must be set before Relay starts. - -```toml -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -version = 1 -mode = "enforce" -priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.openai-default] -model = "provider/efficient-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.openai-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[components.config.targets.responses-default] -model = "provider/efficient-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "https://provider.example.com" - -[components.config.targets.responses-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[components.config.targets.anthropic-default] -model = "provider/efficient-messages" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "https://provider.example.com" - -[components.config.targets.anthropic-default.header_env] -x-api-key = "ANTHROPIC_PROVIDER_API_KEY" - -[components.config.targets.capable-chat] -model = "provider/capable-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.capable-chat.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 3 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" -``` - -## Decision API and Rollout Options - -The following table describes the Decision API and rollout settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `version` | `1` | Component configuration version. | -| `mode` | `"enforce"` | `enforce` applies valid decisions. `observe_only` records the hypothetical decision but dispatches the trusted same-protocol default once. | -| `priority` | `0` | Component execution priority. | -| `decision_api_url` | Required | HTTP or HTTPS URL for the Switchyard Decision API. | -| `decision_profile_id` | Required | Non-empty Switchyard profile identifier sent with each routing request. | -| `decision_timeout_millis` | `25` | Decision API timeout in milliseconds; must be greater than zero. | -| `decision_headers` | Empty | Optional non-sensitive static Decision API headers. | -| `decision_header_env` | Empty | Map of header names to environment-variable names for sensitive Decision API headers. Referenced values must be present and non-blank. | - -Do not define the same header, including case variants, in both a static header -map and its environment map. Keep bearer tokens, API keys, cookies, and other -secrets in environment variables rather than tracked configuration. - -## Request Materialization - -Every Decision API request contains routing identity, inbound protocol, a -request summary, and the attempt number. `request_materialization` controls how -much additional request content is included. - -| Mode | Material sent to Switchyard | -| --- | --- | -| `none` | No `current_request`; use identity, protocol, summary, and attempt metadata only. | -| `summary_only` | No `current_request`; explicitly selects summary-based routing. | -| `latest_user_prompt` | A provider-valid body containing the latest user material plus an explicit prompt field. The request fails open if no user message is available. | -| `recent_message_window` | The most recent normalized messages, bounded by `recent_message_count`, re-encoded for the inbound protocol. | -| `annotated_request` | The final intercepted provider body plus Relay's normalized request representation. | -| `full_body` | The final intercepted provider request body. | - -Choose the least materialization needed by the selected profile. In particular, -`none` and `summary_only` intentionally share the same current wire behavior but -express different configuration intent. - -## Context, Retries, and Inbound Protocols - -The following table describes context, retry, and inbound protocol settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `context_mode` | Required | `payload_only` or `atof_required`. See below. | -| `max_retries` | `3` | Provider retries after the initial attempt; maximum `10`. Each retry obtains a new Switchyard decision. | -| `recent_message_count` | `8` | Message limit for `recent_message_window`; must be greater than zero. | -| `enabled_inbound_profiles` | All three supported protocols | Non-empty list containing `openai_chat`, `openai_responses`, and/or `anthropic_messages`. | -| `atof_endpoint_name` | Required for `atof_required` | Name of exactly one stream sink under the observability component's `atof.sinks` list. | - -`payload_only` supports request-aware routing without an ATOF dependency. -Relay can create request-scoped synthetic identity when stable identity is not -available. - -`atof_required` is for profiles that classify from accumulated ATOF context. It -requires stable session and request identity plus an enabled, authenticated -Relay HTTP ATOF stream sink whose `name` matches `atof_endpoint_name`. The sink -specifies the Switchyard ingestion URL, must use `transport = "http_post"`, and -must set `field_name_policy = "preserve"`. Relay rejects missing or duplicate -named sinks during startup validation. A local ATOF file sink -does not satisfy this delivery requirement by itself. - -## Target Bindings and Trusted Defaults - -The following table describes target bindings and trusted defaults: - -| Option | Required fields | Purpose | -| --- | --- | --- | -| `targets.` | `model`, `protocol`, `endpoint`, `base_url` | Binds an exact Switchyard backend ID to a Relay-owned provider destination. | -| `targets..headers` | None | Optional non-sensitive static provider headers. | -| `targets..header_env` | None | Optional provider header names mapped to environment-variable names. Referenced values must be present and non-blank. | -| `default_targets.openai_chat` | Target using `openai_chat` | Trusted fallback for OpenAI Chat Completions requests. | -| `default_targets.openai_responses` | Target using `openai_responses` | Trusted fallback for OpenAI Responses requests. | -| `default_targets.anthropic_messages` | Target using `anthropic_messages` | Trusted fallback for Anthropic Messages requests. | - -Do not define the same target header, including case variants, in both -`headers` and `header_env`. - -Protocol and endpoint pairs are fixed: - -| Protocol | Endpoint | -| --- | --- | -| `openai_chat` | `/v1/chat/completions` | -| `openai_responses` | `/v1/responses` | -| `anthropic_messages` | `/v1/messages` | - -Each protocol in `enabled_inbound_profiles` requires a corresponding -`default_targets` entry. Defaults for disabled protocols can be omitted. Each -configured default must refer to a target with the matching protocol. Duplicate -exact bindings for model, protocol, endpoint, and base URL are rejected. - -## Recommended Recipes - -### Observe-Only Rollout - -Use observe-only mode to verify decisions and routing marks without sending -traffic to the selected target: - -```toml -[components.config] -mode = "observe_only" -context_mode = "payload_only" -request_materialization = "summary_only" -``` - -Relay calls the Decision API and records the hypothetical selection, but sends -the request to the inbound protocol's trusted default once. Routing retries are -not applied. - -### Payload-Only Request-Aware Routing - -Use recent request content for routing without operating the ATOF accumulator: - -```toml -[components.config] -mode = "enforce" -context_mode = "payload_only" -request_materialization = "recent_message_window" -recent_message_count = 8 -``` - -### ATOF-Backed StageRouter Routing - -Use ATOF context when StageRouter should classify from accumulated session -signals: - -```toml -[components.config] -mode = "enforce" -decision_profile_id = "stage-router" -context_mode = "atof_required" -request_materialization = "summary_only" -atof_endpoint_name = "switchyard" -``` - -Configure the matching observability component with a named stream sink: - -```toml -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 3 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" -``` - -## Runtime and Fail-Open Behavior - -The integration handles routing and failures as follows: - -- Relay validates the decision's backend ID, model, protocol, and endpoint - against its target binding before dispatch. Unknown backends, malformed - decisions, and target drift fail open. -- Decision API errors are not recursively retried. Relay immediately dispatches - the trusted same-protocol default. -- Retryable provider connection, timeout, status, context-window, and - model-unavailable failures can obtain a new decision, bounded by - `max_retries`. -- Non-retryable provider failures use the trusted fallback immediately. After - retry exhaustion, Relay dispatches that fallback once. -- A streaming request can retry only before receiving its first upstream item. - After the stream commits, Relay propagates later failures without another - dispatch. -- Relay forwards provider-specific extensions unchanged when every configured - target uses the inbound protocol. If any target uses another protocol, - nonportable extensions fail open to the trusted same-protocol default before - the Decision API call. Request translation failures before provider dispatch - and buffered response translation failures also fail open. After a streaming - response emits its first item, later provider or translation failures - propagate without another dispatch. -- Routing-mark delivery is best-effort and does not change provider results. - -## Experimental Limitations - -Consider the following experimental limitations before adopting the plugin: - -- The plugin is published as `nemo-relay-switchyard` but is excluded from - default Relay CLI builds. Build the CLI with - `cargo build -p nemo-relay-cli --features switchyard`. -- The routing boundary is currently service-based. Relay calls Switchyard's - HTTP Decision API but does not start or supervise the service. Relay derives - `/health` from `decision_api_url` and fails plugin activation unless the - service returns `{"status":"ok"}`. Decision API failures after activation fail - open to the trusted same-protocol default. -- ATOF-backed profiles additionally depend on Switchyard's HTTP ingestion and - accumulator runtime, stable request and session identity, and one named, - authenticated Relay ATOF HTTP stream sink. A local file sink does not populate - the Switchyard accumulator. -- Provider-protocol translation runs in Relay's process through the - `switchyard-translation` Rust library. Routing decisions and ATOF accumulation - remain out of process. -- The Rust Switchyard component is currently registered only by Relay's CLI - gateway. Hermes native plugins and the LangChain, LangGraph, and Deep Agents - integrations do not load it in process. They receive Switchyard routing only - when their provider traffic is explicitly sent through the Relay gateway. -- The Switchyard examples are manual compatibility and trajectory workflows, - not production deployment orchestration. -- Compatibility is pinned to Switchyard commit - [`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8) - on `topic/nemo-relay-integration`. Testing another revision is deliberate and - can expose contract drift. -- The Decision API and service/library boundary are experimental. -- Nonportable provider extensions in configurations that can translate across - protocols, malformed decisions, service failures, pre-dispatch translation - failures, buffered-response translation failures, and target drift fail open - to Relay's trusted same-protocol default. Failures after a streaming response - commits propagate without another dispatch. -- Secrets must remain environment-referenced. The examples generate ephemeral - local credentials and are not credential-management guidance. - -Run `nemo-relay doctor` to validate component configuration before startup. For -the pinned local service and manual compatibility workflow, refer to the -[Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard). diff --git a/docs/index.yml b/docs/index.yml index fca80db2d..17b77f938 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -41,7 +41,7 @@ navigation: title: "PII Redaction" title-source: frontmatter - folder: ./configure-plugins/switchyard - title: "Switchyard (Experimental)" + title: "Switchyard" title-source: frontmatter - page: "Model Pricing" path: ./configure-plugins/model-pricing.mdx diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md deleted file mode 100644 index 87a2cff68..000000000 --- a/examples/switchyard/README.md +++ /dev/null @@ -1,114 +0,0 @@ - - -# Switchyard Integration Examples - -These examples exercise the experimental NeMo Relay 0.6.0 integration with a separately running -Switchyard Decision API service and the in-process Switchyard translation library. They are -manual, local validation workflows rather than production startup orchestration. - -For the canonical architecture, setup, configuration, validation, and troubleshooting workflow, -refer to the -[Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about). - -## Required Switchyard Revision - -The NeMo Relay 0.6.0 scripts require the following public topic branch and commit: - -```text -https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration -8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Clone the Switchyard repository next to the Relay checkout, then pin the -required commit: - -```bash -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - ../Switchyard-topic-nemo-relay-integration -git -C ../Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Every real-service script verifies this commit before launching `switchyard-server`. To test a -deliberately different checkout, set both variables explicitly: - -```bash -SWITCHYARD_ROOT=/path/to/Switchyard \ -SWITCHYARD_EXPECTED_COMMIT= \ - examples/switchyard/run-real-e2e.sh -``` - -## Examples - -Run these commands from the root of the NeMo Relay checkout. - -The Relay CLI's Switchyard support is compile-time optional. The scripts enable the `switchyard` -feature automatically; custom builds must pass `--features switchyard`. - -### Manual Switchyard Compatibility Smoke Test - -`run-real-e2e.sh` is a manual compatibility smoke test. It starts the pinned Switchyard server, -Relay, and a fake provider, then verifies cold and warm StageRouter decisions, buffered routing, -SSE routing, and the selected model sequence. It is intended to catch cross-repository service -contract regressions; the CI-safe Relay process regression test covers the faster local behavior -checks. The script requires Rust tooling, Python, and `curl`; its temporary logs are removed after -a successful run. - -```bash -examples/switchyard/run-real-e2e.sh -``` - -A successful run ends with: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -### Hermes and Ollama Trajectory - -`run-hermes-ollama-smoke.sh` runs a fixed multi-query trajectory through Hermes, Relay, Ollama, -and Switchyard. It requires Docker, Hermes, and the configured local Ollama models. The script -produces ATOF, ATIF, and OTEL artifacts and can leave Phoenix running with -`SWITCHYARD_KEEP_PHOENIX=1`. - -```bash -examples/switchyard/run-hermes-ollama-smoke.sh -``` - -## Configuration Files - -The directory includes the following configuration and support files: - -- `plugins.toml`: minimal plugin configuration example. -- `real-e2e-plugins.toml` and `real-e2e-profiles.yaml`: deterministic fake-provider E2E. -- `hermes-ollama-plugins.toml` and `hermes-ollama-profiles.yaml`: local Ollama trajectory. -- `fake_upstream.py`: deterministic provider used by the service E2E. -- `otel-collector.yaml`: local OTEL artifact export configuration. - -## Runtime Model - -The scripts launch Switchyard as a separate local process on port `4000`. Relay sends routing -requests to `/v1/routing/decision` and, for ATOF-backed profiles, sends events to -`/v1/atof/events`. Relay owns provider credentials, target bindings, dispatch, retries, and -fallback behavior. Relay executes provider-protocol translation in process through Switchyard's -translation library; the Switchyard service owns ATOF accumulation and routing decisions. The -Switchyard component selects its HTTP ingestion destination by the observability stream sink name, -not by duplicating the sink URL. - -The service is not started automatically by Relay outside these examples. A production deployment -must start a compatible Switchyard service before Relay activates the plugin and configure the -Relay plugin with its Decision API URL. Relay derives the service's root `/health` URL from that -configuration and refuses activation unless it reports `{"status":"ok"}`. - -## Artifacts and Troubleshooting - -Trajectory scripts write to `artifacts/` by default. Set `SWITCHYARD_TRAJECTORY_DIR` to choose a -shareable output directory. On failure, logs are preserved and include the verified Switchyard -revision. Do not place API keys or bearer tokens in configuration files; use environment variables -or an untracked secrets file. For service readiness failures, port conflicts, and the exact -temporary log locations used by `run-real-e2e.sh`, refer to the -[canonical Switchyard 0.6.0 guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about#troubleshooting). diff --git a/examples/switchyard/e2e-common.sh b/examples/switchyard/e2e-common.sh deleted file mode 100755 index bf38627d6..000000000 --- a/examples/switchyard/e2e-common.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Shared process and readiness helpers for the manually-invoked Switchyard E2E scripts. - -e2e_pids=() - -e2e_add_pid() { - e2e_pids+=("$1") -} - -e2e_stop_tree() { - local pid="$1" - local child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - e2e_stop_tree "$child" - done - kill "$pid" 2>/dev/null || true -} - -e2e_stop_processes() { - local pid - for pid in "${e2e_pids[@]}"; do - e2e_stop_tree "$pid" - done - for pid in "${e2e_pids[@]}"; do - wait "$pid" 2>/dev/null || true - done - e2e_pids=() -} - -e2e_wait_for() { - local url="$1" - local attempts="${2:-120}" - local delay="${3:-0.25}" - local process_pid="${4:-}" - local attempt - for attempt in $(seq 1 "$attempts"); do - if [[ -n "$process_pid" ]] && ! kill -0 "$process_pid" 2>/dev/null; then - echo "process $process_pid exited before $url became ready" >&2 - return 1 - fi - if curl --fail --silent "$url" >/dev/null 2>&1; then - return 0 - fi - sleep "$delay" - done - echo "timed out waiting for $url" >&2 - return 1 -} - -e2e_random_token() { - python3 -c 'import secrets; print(secrets.token_hex(24))' -} - -# The experimental Relay integration is tested against a specific public -# Switchyard topic-branch commit. Override this only when intentionally testing -# another revision. -e2e_verify_switchyard_checkout() { - local root="$1" - local expected="${2:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" - local actual - actual="$(git -C "$root" rev-parse HEAD 2>/dev/null)" || { - echo "Switchyard worktree is not a git checkout: $root" >&2 - return 1 - } - if [[ "$actual" != "$expected" ]]; then - echo "Switchyard checkout mismatch: expected $expected, found $actual" >&2 - echo "Set SWITCHYARD_EXPECTED_COMMIT only when intentionally testing another revision." >&2 - return 1 - fi - printf 'Switchyard revision: %s (%s)\n' "$actual" "$(git -C "$root" show -s --format=%s HEAD)" -} - -e2e_tail_logs() { - local directory="$1" - local log - for log in "$directory"/*.log; do - [[ -f "$log" ]] || continue - echo "--- $log" >&2 - tail -100 "$log" >&2 - done -} diff --git a/examples/switchyard/fake_upstream.py b/examples/switchyard/fake_upstream.py deleted file mode 100644 index f27527d14..000000000 --- a/examples/switchyard/fake_upstream.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Deterministic local OpenAI-compatible upstream for Switchyard plugin smoke tests.""" - -import argparse -import json -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - - -class Handler(BaseHTTPRequestHandler): - log_path: Path - - def do_POST(self) -> None: # noqa: N802 - length = int(self.headers.get("content-length", "0")) - body = json.loads(self.rfile.read(length) or b"{}") - with self.log_path.open("a", encoding="utf-8") as output: - output.write(json.dumps({"path": self.path, "body": body}) + "\n") - model = body.get("model", "fake-model") - if body.get("stream"): - chunks = [ - { - "id": "chatcmpl-fake", - "object": "chat.completion.chunk", - "model": model, - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "fake"}, - "finish_reason": None, - } - ], - }, - { - "id": "chatcmpl-fake", - "object": "chat.completion.chunk", - "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - }, - ] - payload = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) - payload += "data: [DONE]\n\n" - encoded = payload.encode() - self.send_response(200) - self.send_header("content-type", "text/event-stream") - self.send_header("content-length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) - return - response = { - "id": "chatcmpl-fake", - "object": "chat.completion", - "model": model, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "fake"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - encoded = json.dumps(response).encode() - self.send_response(200) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(encoded))) - self.end_headers() - self.wfile.write(encoded) - - def log_message(self, _format: str, *_args: object) -> None: - return - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=4101) - parser.add_argument("--log", type=Path, required=True) - args = parser.parse_args() - Handler.log_path = args.log - ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() - - -if __name__ == "__main__": - main() diff --git a/examples/switchyard/hermes-ollama-plugins.toml b/examples/switchyard/hermes-ollama-plugins.toml deleted file mode 100644 index 4878a09ae..000000000 --- a/examples/switchyard/hermes-ollama-plugins.toml +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "hermes-ollama-stage-router" -request_materialization = "full_body" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 120000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "ollama-weak" -openai_responses = "ollama-responses" -anthropic_messages = "ollama-anthropic" - -[components.config.targets.ollama-weak] -model = "llama3.2:latest" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-strong] -model = "qwen3.6:35b" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-responses] -model = "llama3.2:latest" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-anthropic] -model = "llama3.2:latest" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:11434" - -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 3 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" -output_directory = "." -filename = "trajectory.atof.jsonl" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.atif] -enabled = true -agent_name = "Hermes" -agent_version = "switchyard-stage-router-smoke" -model_name = "switchyard-stage-router" -output_directory = "." -filename_template = "trajectory-{session_id}.atif.json" - -[components.config.opentelemetry] -enabled = true - -[[components.config.opentelemetry.endpoints]] -type = "full" -transport = "http_binary" -endpoint = "http://127.0.0.1:4318/v1/traces" -service_name = "nemo-relay-switchyard-e2e" -service_namespace = "switchyard-integration" -instrumentation_scope = "nemo-relay-switchyard" -timeout_millis = 5000 - -[components.config.opentelemetry.endpoints.resource_attributes] -"deployment.environment.name" = "local-e2e" -"switchyard.profile.id" = "hermes-ollama-stage-router" diff --git a/examples/switchyard/hermes-ollama-profiles.yaml b/examples/switchyard/hermes-ollama-profiles.yaml deleted file mode 100644 index 751201851..000000000 --- a/examples/switchyard/hermes-ollama-profiles.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - ollama-weak: - model: llama3.2:latest - format: openai - base_url: http://127.0.0.1:11434/v1 - ollama-strong: - model: qwen3.6:35b - format: openai - base_url: http://127.0.0.1:11434/v1 - -profiles: - hermes-ollama-stage-router: - type: stage_router - capable: ollama-strong - efficient: ollama-weak - fallback_target_on_evict: ollama-strong - picker: efficient_first - # Force the classifier path for ordinary prompts so this smoke demonstrates - # request-complexity routing rather than relying on incidental tool signals. - confidence_threshold: 1.0 - # Each fixed trajectory transition is driven by the latest canonical tool - # result so older critical signals do not leak into the simple follow-up. - signal_recent_window: 1 - classifier: - # Keep classification small and deterministic; qwen3.6:35b remains the - # capable backend selected for the complex trajectory turn. - model: llama3.2:latest - api_key: ${OLLAMA_CLASSIFIER_API_KEY} - base_url: http://127.0.0.1:11434/v1 - timeout_secs: 120.0 - recent_turn_window: 1 - max_tokens: 128 - system_prompt: >- - Select exactly one model tier for the latest user request. Return only - JSON in the form {"tier":"efficient"} or {"tier":"capable"}. Choose efficient - for arithmetic, extraction, formatting, concise factual responses, or - other direct low-risk work. Choose capable for architecture, concurrency, - debugging, multi-step reasoning, subtle correctness constraints, or - requests requiring a defensible technical analysis. In the supplied - State line, severity at least 1.0 always means capable; tests_passed=true - with severity 0.0 means efficient. Classify the latest state even when earlier - conversation turns were more complex. diff --git a/examples/switchyard/otel-collector.yaml b/examples/switchyard/otel-collector.yaml deleted file mode 100644 index 330a87077..000000000 --- a/examples/switchyard/otel-collector.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - -exporters: - file/trajectory: - path: /artifacts/trajectory.otel.json - otlp/phoenix: - endpoint: phoenix:4317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [file/trajectory, otlp/phoenix] diff --git a/examples/switchyard/plugins.toml b/examples/switchyard/plugins.toml deleted file mode 100644 index d6a1a69ed..000000000 --- a/examples/switchyard/plugins.toml +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "smart-stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.openai-default] -model = "weak-model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "weak-model" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "weak-model" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.strong] -model = "strong-model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4102" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 3000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-plugins.toml b/examples/switchyard/real-e2e-plugins.toml deleted file mode 100644 index 541beb057..000000000 --- a/examples/switchyard/real-e2e-plugins.toml +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "remote-stage-router" -request_materialization = "summary_only" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 1000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "weak" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.strong] -model = "provider/strong" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.weak] -model = "provider/weak" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "provider/weak-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "provider/weak-anthropic" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-profiles.yaml b/examples/switchyard/real-e2e-profiles.yaml deleted file mode 100644 index 6127b46ef..000000000 --- a/examples/switchyard/real-e2e-profiles.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:4101/v1 - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:4101/v1 - -profiles: - remote-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: efficient_first - confidence_threshold: 0.7 diff --git a/examples/switchyard/run-hermes-ollama-smoke.sh b/examples/switchyard/run-hermes-ollama-smoke.sh deleted file mode 100755 index f390eb75e..000000000 --- a/examples/switchyard/run-hermes-ollama-smoke.sh +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$relay_root/examples/switchyard/e2e-common.sh" -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" -run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" -artifact_dir="${SWITCHYARD_TRAJECTORY_DIR:-$relay_root/artifacts/hermes-switchyard-$run_id}" -token="$(e2e_random_token)" -docker_network="switchyard-e2e-$run_id" -phoenix_container="switchyard-phoenix-$run_id" -collector_container="switchyard-otel-$run_id" -phoenix_port="${SWITCHYARD_PHOENIX_PORT:-6006}" -keep_phoenix="${SWITCHYARD_KEEP_PHOENIX:-0}" -collector_running=0 -phoenix_running=0 -network_created=0 - -mkdir -p "$artifact_dir/phoenix" -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" >"$artifact_dir/switchyard-revision.txt" - -cleanup() { - local status=$? - e2e_stop_processes - if [[ $collector_running -eq 1 ]]; then - docker rm -f "$collector_container" >/dev/null 2>&1 || true - fi - if [[ $phoenix_running -eq 1 && ( $status -ne 0 || "$keep_phoenix" != "1" ) ]]; then - docker rm -f "$phoenix_container" >/dev/null 2>&1 || true - phoenix_running=0 - fi - if [[ $network_created -eq 1 && $phoenix_running -eq 0 ]]; then - docker network rm "$docker_network" >/dev/null 2>&1 || true - fi - if [[ $status -ne 0 ]]; then - echo "Hermes/StageRouter smoke failed; artifacts preserved in $artifact_dir" >&2 - e2e_tail_logs "$artifact_dir" - fi -} -trap cleanup EXIT - -for dependency in cargo curl docker hermes jq python3 tar; do - command -v "$dependency" >/dev/null || { - echo "missing required command: $dependency" >&2 - exit 1 - } -done - -docker info >/dev/null -for model in llama3.2:latest qwen3.6:35b; do - curl --fail --silent http://127.0.0.1:11434/api/tags \ - | jq -e --arg model "$model" '.models[] | select(.name == $model)' >/dev/null || { - echo "required Ollama model is not installed: $model" >&2 - exit 1 - } -done - -docker network create "$docker_network" >/dev/null -network_created=1 -docker run --detach --rm \ - --name "$phoenix_container" \ - --network "$docker_network" \ - --network-alias phoenix \ - --publish "127.0.0.1:$phoenix_port:6006" \ - --env PHOENIX_WORKING_DIR=/mnt/data \ - --volume "$artifact_dir/phoenix:/mnt/data" \ - arizephoenix/phoenix:13.22 >"$artifact_dir/phoenix.container-id" -phoenix_running=1 -e2e_wait_for "http://127.0.0.1:$phoenix_port/" 240 0.5 - -docker run --detach --rm \ - --name "$collector_container" \ - --network "$docker_network" \ - --publish 127.0.0.1:4318:4318 \ - --volume "$relay_root/examples/switchyard/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \ - --volume "$artifact_dir:/artifacts" \ - otel/opentelemetry-collector-contrib:0.135.0 \ - --config=/etc/otelcol-contrib/config.yaml >"$artifact_dir/collector.container-id" -collector_running=1 - -( - cd "$switchyard_root" - OLLAMA_CLASSIFIER_API_KEY="$token" \ - SWITCHYARD_ATOF_BEARER_TOKEN="$token" \ - cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/hermes-ollama-profiles.yaml" --port 4000 -) >"$artifact_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" -e2e_wait_for http://127.0.0.1:4000/health 240 0.5 - -run_query() { - local sequence="$1" - local label="$2" - local query="$3" - local resume_id="${4:-}" - local -a resume_args=() - local before_lines=0 - local after_lines - local atif_path - if [[ -n "$resume_id" ]]; then - resume_args=(--resume "$resume_id") - fi - if [[ -f "$artifact_dir/trajectory.atof.jsonl" ]]; then - before_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - fi - ( - cd "$artifact_dir" - HERMES_HOME="$artifact_dir/hermes" \ - OPENAI_API_KEY=ollama \ - SWITCHYARD_AUTHORIZATION="Bearer $token" \ - cargo run --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli \ - --features switchyard -- \ - run --agent hermes \ - --plugin-config-path "$relay_root/examples/switchyard/hermes-ollama-plugins.toml" \ - -- chat --provider custom --model llama3.2:latest \ - --query "$query" ${resume_args[@]+"${resume_args[@]}"} \ - --toolsets terminal --quiet --max-turns 2 --ignore-rules - ) >"$artifact_dir/query-$sequence-$label.log" 2>&1 - after_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - printf '%s\t%s\t%s\t%s\n' "$sequence" "$label" "$before_lines" "$after_lines" \ - >> "$artifact_dir/query-event-ranges.tsv" - # Each query is a separate Relay process. Replay its persisted ATOF segment - # as a completion barrier before the next process starts; ingestion is - # idempotent, so events already delivered by the best-effort live exporter - # are reported as duplicates rather than applied twice. - sed -n "$((before_lines + 1)),${after_lines}p" "$artifact_dir/trajectory.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$artifact_dir/trajectory-$sequence-$label.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" - jq -e '(.batch.ingested_events + .batch.duplicate_events) >= 1' \ - "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" >/dev/null - atif_path="$(find "$artifact_dir" -maxdepth 1 -name 'trajectory-*.atif.json' -print \ - | grep -E '/trajectory-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.atif\.json$' \ - | head -1 || true)" - if [[ -z "$atif_path" ]]; then - echo "ATIF exporter did not produce a trajectory for query $sequence" >&2 - exit 1 - fi - mv "$atif_path" "$artifact_dir/trajectory-$sequence-$label.atif.json" -} - -emit_stage_router_signal() { - local sequence="$1" - local label="$2" - local output="$3" - local path="$artifact_dir/trajectory-signal-$sequence-$label.atof.jsonl" - python3 - "$path" "$session_id" "$label" "$output" <<'PY' -import datetime -import json -import pathlib -import sys -import uuid - -path = pathlib.Path(sys.argv[1]) -session_id, label, output = sys.argv[2:] -event_uuid = str(uuid.uuid4()) -base = { - "atof_version": "0.1", - "kind": "scope", - "uuid": event_uuid, - "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "name": "trajectory_fixture", - "category": "tool", - "category_profile": {"tool_call_id": f"fixture-{label}"}, - "metadata": { - "session_id": session_id, - "trajectory_fixture": True, - "trajectory_fixture_label": label, - }, -} -events = [ - {**base, "scope_category": "start", "data": {"label": label}}, - {**base, "scope_category": "end", "data": {"output": output}}, -] -path.write_text("".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events)) -PY - cat "$path" >> "$artifact_dir/trajectory.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$path" > "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" - jq -e '.batch.ingested_events == 2' \ - "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" >/dev/null -} - -simple_query='Return exactly the integer result of 17 + 25, with no explanation.' -complex_query='Do not call tools. Act as a principal concurrency engineer. Analyze a bounded lock-free MPMC queue that uses compare-and-swap on head and tail but no generation counters. Give a concrete ABA failure interleaving, identify the required C++ memory order on each publication and consumption edge, and propose the smallest defensible correction. Be precise about the linearization points.' -followup_query='Ignore the earlier technical topic. Reply with exactly SIMPLE_DONE and nothing else.' - -run_query 01 simple "$simple_query" -session_id="$(jq -r 'select(.name == "switchyard.routing.requested") | .data.session_id' "$artifact_dir/trajectory.atof.jsonl" | head -1)" -if [[ -z "$session_id" || "$session_id" == "null" ]]; then - echo "could not recover the Hermes session ID from the first routing mark" >&2 - exit 1 -fi -emit_stage_router_signal 02 complex \ - 'CUDA out of memory while analyzing the concurrent queue; critical failure requires careful recovery and a capable model.' -run_query 02 complex "$complex_query" "$session_id" -emit_stage_router_signal 03 simple-followup \ - 'All tests passed. The next request is a direct low-risk formatting response: return exactly SIMPLE_DONE.' -run_query 03 simple-followup "$followup_query" "$session_id" - -# Give the asynchronous OTLP exporter a short flush window, then stop the -# collector cleanly so its file exporter closes the shareable OTLP JSON file. -sleep 2 -docker stop --time 10 "$collector_container" >/dev/null -collector_running=0 - -python3 - "$artifact_dir" "$session_id" "$phoenix_port" "$simple_query" "$complex_query" "$followup_query" <<'PY' -import json -import pathlib -import sys - -root = pathlib.Path(sys.argv[1]) -session_id = sys.argv[2] -phoenix_port = sys.argv[3] -queries = sys.argv[4:] -atof_path = root / "trajectory.atof.jsonl" -events = [json.loads(line) for line in atof_path.read_text().splitlines() if line.strip()] -marks = [event for event in events if event.get("name", "").startswith("switchyard.routing.")] -decisions = [event for event in marks if event.get("name") == "switchyard.routing.decision"] - -expected_models = ["llama3.2:latest", "qwen3.6:35b", "llama3.2:latest"] -event_ranges = [] -for line in (root / "query-event-ranges.tsv").read_text().splitlines(): - sequence, label, start, end = line.split("\t") - event_ranges.append((sequence, label, int(start), int(end))) -representative_decisions = [] -for sequence, label, start, end in event_ranges: - segment = events[start:end] - decision = next( - (event for event in segment if event.get("name") == "switchyard.routing.decision"), - None, - ) - if decision is None: - raise SystemExit(f"query {sequence} ({label}) produced no successful routing decision") - representative_decisions.append(decision) -actual_models = [event.get("data", {}).get("selected_model") for event in representative_decisions] -if actual_models != expected_models: - raise SystemExit(f"unexpected StageRouter route sequence: {actual_models}; expected {expected_models}") - -required_mark_names = {"switchyard.routing.requested", "switchyard.routing.decision"} -if not required_mark_names.issubset({event.get("name") for event in marks}): - raise SystemExit("routing requested/decision marks were not both emitted") -for event in marks: - name = event.get("name") - if event.get("category") != "custom": - raise SystemExit(f"{name} did not use category=custom") - if event.get("category_profile", {}).get("subtype") != name: - raise SystemExit(f"{name} category_profile.subtype was not canonical") - schema = event.get("data_schema", {}) - if schema != {"name": "switchyard.routing_mark", "version": "1"}: - raise SystemExit(f"{name} had unexpected data_schema: {schema}") - metadata = event.get("metadata", {}) - if metadata.get("session_id") != session_id: - raise SystemExit(f"{name} did not mirror session identity") -for event in decisions: - data = event["data"] - for key in ("decision_id", "router", "routing_attempt", "backend_id", "selected_tier", "selected_model", "latency_ms", "rollout_mode"): - if key not in data or data[key] is None: - raise SystemExit(f"decision mark missing {key}: {data}") - -atif_paths = sorted(root.glob("trajectory-*.atif.json")) -if len(atif_paths) != 3: - raise SystemExit(f"expected three ATIF trajectories, found {len(atif_paths)}") -for path in atif_paths: - payload = json.loads(path.read_text()) - if not payload.get("steps"): - raise SystemExit(f"ATIF trajectory has no steps: {path.name}") - -otel_path = root / "trajectory.otel.json" -if not otel_path.exists() or not otel_path.read_text().strip(): - raise SystemExit("OTLP file exporter did not produce trajectory.otel.json") -otel_batches = [json.loads(line) for line in otel_path.read_text().splitlines() if line.strip()] - -summary = { - "session_id": session_id, - "queries": [{ - "sequence": index + 1, - "input": query, - "selected_model": actual_models[index], - "selected_tier": representative_decisions[index]["data"].get("selected_tier"), - "reason_code": representative_decisions[index]["data"].get("reason_code"), - "reason_summary": representative_decisions[index]["data"].get("reason_summary"), - } for index, query in enumerate(queries)], - "expected_route_sequence": expected_models, - "actual_route_sequence": actual_models, - "routing_basis": [ - "cold StageRouter efficient default", - "canonical ATOF critical-error tool result (capable override)", - "canonical ATOF clean-tests tool result (efficient classifier decision)", - ], - "atof": {"file": atof_path.name, "event_count": len(events), "routing_mark_count": len(marks)}, - "atif": {"files": [path.name for path in atif_paths], "trajectory_count": len(atif_paths)}, - "otel": {"file": otel_path.name, "export_batch_count": len(otel_batches)}, - "phoenix_url": f"http://127.0.0.1:{phoenix_port}", -} -(root / "trajectory-summary.json").write_text(json.dumps(summary, indent=2) + "\n") -readme = f"""# Hermes / Ollama / Switchyard StageRouter trajectory - -This bundle captures one fixed three-query Hermes session routed through NeMo -Relay and the Switchyard Decision API. The verified representative route is: - -1. `llama3.2:latest` (efficient) — cold StageRouter default -2. `qwen3.6:35b` (capable) — critical-signal StageRouter override -3. `llama3.2:latest` (efficient) — clean-state classifier decision - -Session ID: `{session_id}` - -## Important fixture note - -The `CUDA out of memory` text in `trajectory-signal-02-complex.atof.jsonl` is an -intentional, synthetic ATOF tool-result fixture. The machine did not run out of -memory. Fixture events carry `metadata.trajectory_fixture = true` and a fixture -label so they cannot be confused with organic Hermes events. - -The fixtures are necessary for this demonstration because the current -Switchyard StageRouter Decision API classifies from its accumulated ATOF snapshot, -not directly from `current_request.body`. The critical fixture exercises the -real capable override. The clean-tests fixture removes the prior critical signal -from the one-result window and exercises the efficient classifier path. - -## File map - -| Files | Contents | Test coverage | -| --- | --- | --- | -| `trajectory-summary.json` | Machine-readable queries, selected models, reasons, counts, and Phoenix URL | Confirms expected and actual representative routes match | -| `trajectory.atof.jsonl` | Complete Relay ATOF stream for all three queries and the labeled fixtures | Identity propagation, lifecycle events, routing marks, and accumulator input | -| `trajectory-01-simple.atof.jsonl` | ATOF emitted by the first Hermes invocation | Cold-start efficient default | -| `trajectory-02-complex.atof.jsonl` | ATOF emitted by the complex Hermes invocation | Dispatch through the selected capable backend | -| `trajectory-03-simple-followup.atof.jsonl` | ATOF emitted by the final Hermes invocation | Return to the efficient backend | -| `trajectory-signal-*.atof.jsonl` | Canonical, labeled tool start/end fixtures | Capable critical-error override and efficient clean-state classification | -| `*.atof-ingest.json` | Switchyard ingestion reports for query segments and fixtures | Successful or idempotent ATOF accumulation | -| `trajectory-01-simple.atif.json` | ATIF representation of query 1 | Efficient-model trajectory structure | -| `trajectory-02-complex.atif.json` | ATIF representation of query 2 | Capable-model trajectory structure | -| `trajectory-03-simple-followup.atif.json` | ATIF representation of query 3 | Efficient follow-up trajectory structure | -| `trajectory.otel.json` | OTLP JSON batches written by the OpenTelemetry Collector | Relay spans exported to the collector and forwarded to Phoenix | -| `query-*.log` | Hermes/Relay stdout and stderr for each invocation | Human-readable harness responses and execution diagnostics | -| `query-event-ranges.tsv` | Query label and ATOF line-count boundaries | Separates representative user-query decisions from extra Hermes calls | - -## Routing-mark assertions - -The smoke validates every `switchyard.routing.*` mark in the cumulative ATOF -stream. Each mark must have: - -- `category: "custom"` -- `category_profile.subtype` equal to the mark name -- `data_schema.name: "switchyard.routing_mark"` -- `data_schema.version: "1"` -- the expected session identity in metadata - -Decision marks must also include a decision ID, router, attempt, backend, tier, -model, latency, and rollout mode. This run produced {len(marks)} routing marks -across {len(events)} total ATOF events. - -## Phoenix and OTLP - -During the smoke, Relay sends OTLP/HTTP to a local OpenTelemetry Collector. The -collector writes `trajectory.otel.json` and forwards the same spans to Phoenix. -The run produced {len(otel_batches)} OTLP export batches. When the smoke is run -with `SWITCHYARD_KEEP_PHOENIX=1`, open the `phoenix_url` from -`trajectory-summary.json` before stopping the printed Phoenix container. - -## Reproduce - -Install both `llama3.2:latest` and `qwen3.6:35b` in Ollama, ensure the cumulative -Switchyard checkout is available, then run: - -```bash -SWITCHYARD_KEEP_PHOENIX=1 examples/switchyard/run-hermes-ollama-smoke.sh -``` - -The script validates the route, mark shape, ATIF contents, and OTLP output before -creating this bundle. -""" -(root / "TRAJECTORY_README.md").write_text(readme) -print(json.dumps(summary, indent=2)) -PY - -( - cd "$artifact_dir" - tar -czf trajectory-bundle.tar.gz \ - TRAJECTORY_README.md \ - trajectory-summary.json \ - trajectory.atof.jsonl \ - trajectory.otel.json \ - trajectory-*.atif.json \ - trajectory-*.atof.jsonl \ - trajectory-*.atof-ingest.json \ - query-*.log \ - query-event-ranges.tsv -) - -echo "Hermes/Ollama StageRouter trajectory passed: llama3.2 -> qwen3.6:35b -> llama3.2" -echo "Artifacts: $artifact_dir" -echo "Bundle: $artifact_dir/trajectory-bundle.tar.gz" -if [[ "$keep_phoenix" == "1" ]]; then - echo "Phoenix: http://127.0.0.1:$phoenix_port (container $phoenix_container left running)" -else - echo "Set SWITCHYARD_KEEP_PHOENIX=1 to leave Phoenix running after the smoke." -fi diff --git a/examples/switchyard/run-real-e2e.sh b/examples/switchyard/run-real-e2e.sh deleted file mode 100755 index 1f6810871..000000000 --- a/examples/switchyard/run-real-e2e.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$relay_root/examples/switchyard/e2e-common.sh" -relay_toolchain="" -if command -v rustup >/dev/null 2>&1; then - relay_toolchain="$(cd "$relay_root" && rustup show active-toolchain)" - relay_toolchain="${relay_toolchain%% *}" -fi -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" -work_dir="$(mktemp -d)" -upstream_log="$work_dir/upstream.jsonl" -token="$(e2e_random_token)" - -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" - -cleanup() { - local status=$? - e2e_stop_processes - if [[ $status -eq 0 ]]; then - rm -rf "$work_dir" - else - echo "E2E logs preserved in $work_dir" >&2 - e2e_tail_logs "$work_dir" - fi -} -trap cleanup EXIT - -python3 "$relay_root/examples/switchyard/fake_upstream.py" \ - --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & -e2e_add_pid "$!" - -( - cd "$switchyard_root" - SWITCHYARD_ATOF_BEARER_TOKEN="$token" cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/real-e2e-profiles.yaml" --port 4000 -) >"$work_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" - -e2e_wait_for http://127.0.0.1:4000/health - -( - if [[ -n "$relay_toolchain" ]]; then - export RUSTUP_TOOLCHAIN="$relay_toolchain" - fi - cd "$work_dir" - SWITCHYARD_AUTHORIZATION="Bearer $token" cargo run \ - --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli --features switchyard -- \ - --plugin-config-path "$relay_root/examples/switchyard/real-e2e-plugins.toml" \ - --bind 127.0.0.1:4041 -) >"$work_dir/relay.log" 2>&1 & -relay_pid="$!" -e2e_add_pid "$relay_pid" - -e2e_wait_for http://127.0.0.1:4041/healthz 120 0.25 "$relay_pid" - -request() { - local request_id="$1" - local stream="$2" - curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ - -H 'content-type: application/json' \ - -H 'x-nemo-relay-session-id: e2e-session' \ - -H "x-nemo-relay-request-id: $request_id" \ - --data-binary "{\"model\":\"client-model\",\"stream\":$stream,\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}]}" -} - -request cold-request false >"$work_dir/cold.json" - -for payload in \ - '{"hook_event_name":"on_session_start","session_id":"e2e-session"}' \ - '{"hook_event_name":"pre_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' \ - '{"hook_event_name":"post_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"tool_response":{"output":"CUDA out of memory"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' -do - curl --fail --silent http://127.0.0.1:4041/hooks/hermes \ - -H 'content-type: application/json' --data-binary "$payload" >/dev/null -done - -sleep 1 -request warm-request false >"$work_dir/warm.json" -request stream-request true >"$work_dir/stream.sse" - -python3 - "$upstream_log" "$work_dir/stream.sse" <<'PY' -import json -import pathlib -import sys - -records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] -models = [record["body"]["model"] for record in records] -if models != ["provider/weak", "provider/strong", "provider/strong"]: - raise SystemExit(f"unexpected cold/warm/stream route sequence: {models}") -stream = pathlib.Path(sys.argv[2]).read_text() -if "fake" not in stream or "[DONE]" not in stream: - raise SystemExit(f"unexpected SSE output: {stream}") -print(f"real Switchyard E2E passed: {models}") -PY diff --git a/justfile b/justfile index fcfff98e4..5db79ee54 100644 --- a/justfile +++ b/justfile @@ -470,7 +470,6 @@ local_dependencies = ( "nemo-relay-plugin", "nemo-relay-adaptive", "nemo-relay-pii-redaction", - "nemo-relay-switchyard", "nemo-relay-ffi", "nemo-relay-cli", ) @@ -761,7 +760,6 @@ published_cargo_packages() { nemo-relay \ nemo-relay-adaptive \ nemo-relay-pii-redaction \ - nemo-relay-switchyard \ nemo-relay-ffi \ nemo-relay-cli } @@ -1530,11 +1528,6 @@ package-rust: cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') ;; - nemo-relay-switchyard) - cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') - cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"') - cargo_package_config+=(--config 'patch.crates-io.nemo-relay-plugin.path="crates/plugin"') - ;; nemo-relay-ffi|nemo-relay-cli) cargo_package_config+=(--config 'patch.crates-io.nemo-relay-types.path="crates/types"') cargo_package_config+=(--config 'patch.crates-io.nemo-relay.path="crates/core"')