From 72185ce9dbed2e821cb8663d2ee9f33b82e5f5f8 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 29 Jul 2026 18:38:19 -0600 Subject: [PATCH 1/5] feat: route with Switchyard libsy in process Signed-off-by: Bryan Bednarski --- ATTRIBUTIONS-Rust.md | 718 +++++- Cargo.lock | 92 +- Cargo.toml | 4 +- crates/cli/src/plugins/editor_model.rs | 16 +- crates/cli/src/server/mod.rs | 18 +- crates/cli/tests/switchyard_process_e2e.rs | 154 +- crates/switchyard/Cargo.toml | 11 +- crates/switchyard/LIBSY_GAPS.md | 116 + crates/switchyard/README.md | 149 +- crates/switchyard/src/component.rs | 1912 +++++---------- crates/switchyard/src/contract.rs | 187 -- crates/switchyard/src/lib.rs | 8 +- crates/switchyard/src/stream_translation.rs | 430 +++- crates/switchyard/src/translation.rs | 445 ++-- crates/switchyard/tests/contract.rs | 79 - .../switchyard/tests/unit/component_tests.rs | 2091 ++++------------- docs/configure-plugins/about.mdx | 4 +- docs/configure-plugins/switchyard/about.mdx | 206 +- .../switchyard/configuration.mdx | 406 +--- examples/switchyard/README.md | 116 +- examples/switchyard/e2e-common.sh | 23 - examples/switchyard/fake_upstream.py | 3 + .../switchyard/hermes-ollama-plugins.toml | 96 - .../switchyard/hermes-ollama-profiles.yaml | 45 - examples/switchyard/otel-collector.yaml | 22 - examples/switchyard/plugins.toml | 68 +- examples/switchyard/real-e2e-plugins.toml | 73 - examples/switchyard/real-e2e-profiles.yaml | 21 - .../switchyard/run-hermes-ollama-smoke.sh | 409 ---- examples/switchyard/run-real-e2e.sh | 82 +- 30 files changed, 2801 insertions(+), 5203 deletions(-) create mode 100644 crates/switchyard/LIBSY_GAPS.md delete mode 100644 crates/switchyard/src/contract.rs delete mode 100644 crates/switchyard/tests/contract.rs delete mode 100644 examples/switchyard/hermes-ollama-plugins.toml delete mode 100644 examples/switchyard/hermes-ollama-profiles.yaml delete mode 100644 examples/switchyard/otel-collector.yaml delete mode 100644 examples/switchyard/real-e2e-plugins.toml delete mode 100644 examples/switchyard/real-e2e-profiles.yaml delete mode 100755 examples/switchyard/run-hermes-ollama-smoke.sh diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index 2821c2c15..b580775eb 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -24920,9 +24920,8 @@ SOFTWARE. ## md-5 - 0.10.6 **Repository URL**: https://github.com/RustCrypto/hashes -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -25125,37 +25124,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -Copyright (c) 2006-2009 Graydon Hoare -Copyright (c) 2009-2013 Mozilla Foundation -Copyright (c) 2016 Artyom Pavlov - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## memchr - 2.8.0 @@ -28444,6 +28413,87 @@ limitations under the License. ``` +## opentelemetry - 0.32.0 +**Repository URL**: https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + ## opentelemetry-http - 0.31.0 **Repository URL**: https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-http **License Type(s)**: Apache-2.0 @@ -34023,7 +34073,7 @@ limitations under the License. ``` -## rand - 0.9.3 +## rand - 0.8.7 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html @@ -34104,7 +34154,7 @@ limitations under the License. ``` -## rand_chacha - 0.9.0 +## rand - 0.9.3 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html @@ -34185,8 +34235,8 @@ limitations under the License. ``` -## rand_core - 0.10.1 -**Repository URL**: https://github.com/rust-random/rand_core +## rand_chacha - 0.3.1 +**Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html ``` @@ -34378,12 +34428,107 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + ``` -## rand_core - 0.9.5 +## rand_chacha - 0.9.0 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + +## rand_core - 0.10.1 +**Repository URL**: https://github.com/rust-random/rand_core +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -34575,34 +34720,424 @@ APPENDIX: How to apply the Apache License to your work. ``` -## rcgen - 0.13.2 -**Repository URL**: https://github.com/rustls/rcgen +## rand_core - 0.6.4 +**Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html ``` -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +``` + +## rand_core - 0.9.5 +**Repository URL**: https://github.com/rust-random/rand +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +``` + +## rcgen - 0.13.2 +**Repository URL**: https://github.com/rustls/rcgen +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." @@ -43391,6 +43926,87 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## switchyard-libsy - 0.1.0 +**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + ## switchyard-protocol - 0.1.0 **Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard **License Type(s)**: Apache-2.0 diff --git a/Cargo.lock b/Cargo.lock index 1ed2ac773..8fc888200 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1614,7 +1614,7 @@ dependencies = [ "nemo-relay-worker-proto", "object_store", "openinference-semantic-conventions", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry-otlp", "opentelemetry_sdk", "reqwest", @@ -1685,7 +1685,7 @@ dependencies = [ "nemo-relay-adaptive", "nemo-relay-pii-redaction", "nemo-relay-switchyard", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry_sdk", "percent-encoding", "rcgen", @@ -1799,17 +1799,16 @@ dependencies = [ name = "nemo-relay-switchyard" version = "0.6.0" dependencies = [ - "async-stream", - "axum", "futures-util", + "http", "nemo-relay", - "reqwest", "schemars", "serde", "serde_json", + "switchyard-libsy", + "switchyard-protocol", "switchyard-translation", "tokio", - "uuid", ] [[package]] @@ -2005,7 +2004,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea63d12c54e611c8b28e031ac1d6d8778c7f399dc1f661da9260a543676a07d" dependencies = [ - "opentelemetry", + "opentelemetry 0.31.0", ] [[package]] @@ -2028,6 +2027,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "js-sys", +] + [[package]] name = "opentelemetry-http" version = "0.31.0" @@ -2037,7 +2045,7 @@ dependencies = [ "async-trait", "bytes", "http", - "opentelemetry", + "opentelemetry 0.31.0", "reqwest", ] @@ -2048,7 +2056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ "http", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", @@ -2065,7 +2073,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry_sdk", "prost", "tonic", @@ -2081,7 +2089,7 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.31.0", "percent-encoding", "rand 0.9.3", "thiserror 2.0.18", @@ -2546,13 +2554,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2567,6 +2586,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2577,6 +2606,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -3289,22 +3327,44 @@ dependencies = [ "sval_nested", ] +[[package]] +name = "switchyard-libsy" +version = "0.1.0" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" +dependencies = [ + "async-trait", + "futures", + "opentelemetry 0.32.0", + "parking_lot", + "rand 0.8.7", + "serde", + "serde_json", + "switchyard-protocol", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "switchyard-protocol" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80192a3665425e92b4e10df3bff748cc60180cd028ba207bd70fb95df6869d5" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" dependencies = [ + "async-trait", + "futures", "serde", "serde_json", + "thiserror 2.0.18", ] [[package]] name = "switchyard-translation" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c977a683d8921e1ee7f01373162e14cbbfac28ef2a87dde8441c1691b2fed9" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" dependencies = [ + "async-stream", + "futures", "serde", "serde_json", "switchyard-protocol", diff --git a/Cargo.toml b/Cargo.toml index c561b1d88..bb29ad13e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,9 @@ nemo-relay-worker = { version = "0.6.0", path = "crates/worker" } nemo-relay-adaptive = { version = "0.6.0", path = "crates/adaptive" } nemo-relay-pii-redaction = { version = "0.6.0", path = "crates/pii-redaction" } nemo-relay-switchyard = { version = "0.6.0", path = "crates/switchyard" } -switchyard-translation = "0.1.0" +switchyard-libsy = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } +switchyard-protocol = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } +switchyard-translation = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } nemo-relay-ffi = { version = "0.6.0", path = "crates/ffi" } nemo-relay-cli = { version = "0.6.0", path = "crates/cli" } opentelemetry = { version = "0.31", default-features = false } diff --git a/crates/cli/src/plugins/editor_model.rs b/crates/cli/src/plugins/editor_model.rs index 5fa842616..0680f6586 100644 --- a/crates/cli/src/plugins/editor_model.rs +++ b/crates/cli/src/plugins/editor_model.rs @@ -15,7 +15,7 @@ use nemo_relay_adaptive::AdaptiveConfig; use nemo_relay_adaptive::plugin_component::ADAPTIVE_PLUGIN_KIND; use nemo_relay_pii_redaction::component::{PII_REDACTION_PLUGIN_KIND, PiiRedactionConfig}; #[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; +use nemo_relay_switchyard::{AlgorithmConfig, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::{Map, Value, json}; @@ -52,7 +52,7 @@ impl EditableComponent { Self::NemoGuardrails(_) => "NeMo Guardrails", Self::PiiRedaction(_) => "PII Redaction", #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard Decision API", + Self::Switchyard(_) => "Switchyard libsy", } } @@ -1174,20 +1174,18 @@ pub(super) fn pii_redaction_summary(state: &ComponentEditorState bool { - !config.decision_profile_id.is_empty() || !config.targets.is_empty() + !config.targets.is_empty() } #[cfg(feature = "switchyard")] pub(super) fn switchyard_summary(state: &ComponentEditorState) -> String { - let profile = if state.config.decision_profile_id.is_empty() { - "unconfigured" - } else { - state.config.decision_profile_id.as_str() + let algorithm = match &state.config.algorithm { + AlgorithmConfig::Random { .. } => "random", }; format!( - "component {}, profile {}, targets {}", + "component {}, algorithm {}, targets {}", if state.enabled { "enabled" } else { "disabled" }, - profile, + algorithm, state.config.targets.len() ) } diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index a263f0a54..e0db8a56a 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -29,9 +29,7 @@ use nemo_relay::plugin::{ use nemo_relay_adaptive::plugin_component::register_adaptive_component; use nemo_relay_pii_redaction::component::register_pii_redaction_component; #[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{ - register_switchyard_component, validate_switchyard_atof_configuration, -}; +use nemo_relay_switchyard::register_switchyard_component; use reqwest::Client; use serde_json::Value; use subtle::ConstantTimeEq; @@ -894,8 +892,6 @@ pub(crate) enum PluginComponentSetupError { PiiRedaction(String), #[cfg(feature = "switchyard")] Switchyard(String), - #[cfg(feature = "switchyard")] - SwitchyardAtof(String), } impl PluginComponentSetupError { @@ -905,8 +901,6 @@ impl PluginComponentSetupError { Self::PiiRedaction(_) => "PII redaction plugin", #[cfg(feature = "switchyard")] Self::Switchyard(_) => "Switchyard plugin", - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(_) => "Switchyard ATOF", } } @@ -917,8 +911,6 @@ impl PluginComponentSetupError { } #[cfg(feature = "switchyard")] Self::Switchyard(error) => format!("registration failed: {error}"), - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => error.clone(), } } } @@ -939,10 +931,6 @@ impl std::fmt::Display for PluginComponentSetupError { Self::Switchyard(error) => { write!(formatter, "Switchyard plugin registration failed: {error}") } - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => { - write!(formatter, "Switchyard ATOF validation failed: {error}") - } } } } @@ -961,10 +949,6 @@ pub(crate) fn register_and_validate_plugin_components( if let Err(error) = register_switchyard_component() { errors.push(PluginComponentSetupError::Switchyard(error.to_string())); } - #[cfg(feature = "switchyard")] - if let Err(error) = validate_switchyard_atof_configuration(_plugin_config) { - errors.push(PluginComponentSetupError::SwitchyardAtof(error)); - } errors } diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs index a9ba3397d..da78abed3 100644 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ b/crates/cli/tests/switchyard_process_e2e.rs @@ -11,7 +11,7 @@ use axum::body::Body; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use axum::routing::{get, post}; +use axum::routing::post; use axum::{Json, Router}; use serde_json::{Value, json}; @@ -28,53 +28,6 @@ impl Drop for ChildGuard { } } -#[derive(Clone, Default)] -struct DecisionState { - requests: Arc>>, -} - -async fn decide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let call = { - let mut requests = state.requests.lock().unwrap(); - requests.push((headers, request)); - requests.len() - }; - if call == 4 { - return Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .body(Body::from("decision API unavailable")) - .unwrap(); - } - let body = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": format!("decision-{call}"), - "router": {"name": "fake-ci-router", "version": "1"}, - "route": { - "tier": "strong", - "target_model": "provider/selected", - "backend_id": "selected-chat", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.99, - "reason_code": "ci_fixture", - "reason_summary": "deterministic process E2E decision" - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn switchyard_health() -> Json { - Json(json!({"status": "ok"})) -} - #[derive(Clone, Default)] struct ProviderState { requests: Arc>>, @@ -89,9 +42,7 @@ async fn provide( let model = request["model"].as_str().unwrap_or("unknown").to_string(); let malformed_response = !stream && model == "provider/selected" - && headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response"); + && request["messages"][0]["content"] == "malformed-response"; state.requests.lock().unwrap().push((headers, request)); if malformed_response { return Response::builder() @@ -103,6 +54,7 @@ async fn provide( if stream { let first = json!({ "id": "chat-ci", "object": "chat.completion.chunk", "model": model, + "system_fingerprint": "fp_process_e2e", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}, "finish_reason": null}] }); let last = json!({ @@ -162,17 +114,7 @@ async fn wait_for_gateway(client: &reqwest::Client, url: &str, child: &mut Child } #[tokio::test(flavor = "multi_thread")] -async fn switchyard_plugin_routes_buffered_and_streaming_then_fails_open() { - let decision_state = DecisionState::default(); - let decision_requests = Arc::clone(&decision_state.requests); - let (decision_url, decision_task) = start_server( - Router::new() - .route("/v1/routing/decision", post(decide)) - .route("/health", get(switchyard_health)) - .with_state(decision_state), - ) - .await; - +async fn switchyard_plugin_routes_buffered_and_streaming_without_a_service() { let provider_state = ProviderState::default(); let provider_requests = Arc::clone(&provider_state.requests); let (provider_url, provider_task) = start_server( @@ -192,42 +134,30 @@ kind = "switchyard" enabled = true [components.config] -mode = "enforce" -decision_api_url = "{decision_url}/v1/routing/decision" -decision_profile_id = "ci-process-e2e" -request_materialization = "full_body" -context_mode = "payload_only" -decision_timeout_millis = 1000 +version = 2 max_retries = 0 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] openai_chat = "fallback-chat" -openai_responses = "fallback-responses" -anthropic_messages = "fallback-anthropic" [components.config.targets.selected-chat] model = "provider/selected" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "{provider_url}" +weight = 1 [components.config.targets.fallback-chat] model = "provider/fallback" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "{provider_url}" - -[components.config.targets.fallback-responses] -model = "provider/fallback" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "{provider_url}" - -[components.config.targets.fallback-anthropic] -model = "provider/fallback" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "{provider_url}" +weight = 0 "# ); std::fs::write(&config_path, config).unwrap(); @@ -251,6 +181,7 @@ base_url = "{provider_url}" let send_chat = |request_id: &'static str, stream: bool| { client .post(format!("{gateway_url}/v1/chat/completions")) + .header("authorization", "Bearer caller-secret") .header("x-nemo-relay-session-id", "ci-process-session") .header("x-nemo-relay-request-id", request_id) .header( @@ -261,7 +192,7 @@ base_url = "{provider_url}" .json(&json!({ "model": "client/model", "stream": stream, - "messages": [{"role": "user", "content": "process boundary test"}] + "messages": [{"role": "user", "content": request_id}] })) .send() }; @@ -271,56 +202,18 @@ base_url = "{provider_url}" let buffered: Value = buffered.json().await.unwrap(); assert_eq!(buffered["model"], "provider/selected"); - let translated = client - .post(format!("{gateway_url}/v1/responses")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", "translated-request") - .json(&json!({ - "model": "client/model", - "stream": false, - "input": "process boundary response translation" - })) - .send() - .await - .unwrap(); - assert!(translated.status().is_success()); - let translated: Value = translated.json().await.unwrap(); - assert_eq!(translated["object"], "response"); - let streaming = send_chat("stream-request", true).await.unwrap(); assert!(streaming.status().is_success()); let streaming = streaming.text().await.unwrap(); assert!(streaming.contains("streamed")); + assert!(streaming.contains("fp_process_e2e")); assert!(streaming.contains("[DONE]")); - let fallback = send_chat("fallback-request", false).await.unwrap(); - assert!(fallback.status().is_success()); - let fallback: Value = fallback.json().await.unwrap(); - assert_eq!(fallback["model"], "provider/fallback"); - let malformed = send_chat("malformed-response", false).await.unwrap(); assert!(malformed.status().is_success()); let malformed: Value = malformed.json().await.unwrap(); assert_eq!(malformed["model"], "provider/fallback"); - let decisions = decision_requests.lock().unwrap(); - assert_eq!(decisions.len(), 5); - for (headers, body) in decisions.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - assert_eq!( - headers - .get("x-nemo-relay-session-id") - .unwrap() - .to_str() - .unwrap(), - "ci-process-session" - ); - assert_eq!(body["schema_version"], "switchyard.routing_request.v1"); - assert_eq!(body["decision_profile"]["profile_id"], "ci-process-e2e"); - } - drop(decisions); - let providers = provider_requests.lock().unwrap(); let models = providers .iter() @@ -332,25 +225,12 @@ base_url = "{provider_url}" "provider/selected", "provider/selected", "provider/selected", - "provider/fallback", - "provider/selected", "provider/fallback" ] ); - assert!(providers[1].1["messages"].is_array()); - assert!(providers[1].1.get("input").is_none()); - assert_eq!(providers[1].1["messages"][0]["role"], "user"); - assert_eq!( - providers[1].1["messages"][0]["content"], - "process boundary response translation" - ); let malformed_models = providers .iter() - .filter(|(headers, _)| { - headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response") - }) + .filter(|(_, body)| body["messages"][0]["content"] == "malformed-response") .map(|(_, body)| body["model"].as_str().unwrap()) .collect::>(); assert_eq!( @@ -360,8 +240,8 @@ base_url = "{provider_url}" for (headers, _) in providers.iter() { assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); + assert!(!headers.contains_key("authorization")); } - decision_task.abort(); provider_task.abort(); } diff --git a/crates/switchyard/Cargo.toml b/crates/switchyard/Cargo.toml index 7e7466884..77a68ee8f 100644 --- a/crates/switchyard/Cargo.toml +++ b/crates/switchyard/Cargo.toml @@ -7,7 +7,7 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true -description = "First-party Switchyard Decision API routing plugin for NeMo Relay." +description = "First-party in-process Switchyard libsy routing plugin for NeMo Relay." readme = "README.md" [lints] @@ -21,14 +21,13 @@ schema = ["dep:schemars", "nemo-relay/schema"] nemo-relay.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" +switchyard-libsy.workspace = true +switchyard-protocol.workspace = true switchyard-translation.workspace = true schemars = { version = "0.8", optional = true } -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } -tokio = { version = "1", default-features = false, features = ["time"] } -uuid = { workspace = true, features = ["v7"] } +tokio = { version = "1", default-features = false, features = ["rt", "sync"] } futures-util = "0.3" -async-stream = "0.3" +http = "1" [dev-dependencies] -axum = "0.8" tokio = { version = "1", features = ["net", "rt", "macros", "sync", "test-util", "rt-multi-thread", "time"] } diff --git a/crates/switchyard/LIBSY_GAPS.md b/crates/switchyard/LIBSY_GAPS.md new file mode 100644 index 000000000..860ca7bdf --- /dev/null +++ b/crates/switchyard/LIBSY_GAPS.md @@ -0,0 +1,116 @@ + + +# libsy Contract Assessment + +This assessment covers the Relay-native random-router integration driven +through `Algorithm::run_stream`. Relay performs provider calls and returns each +real response, stream, or error through `CallLlmRequest::respond`. + +The initial assessment uses Switchyard revision +`c8ca731adec50c948739a71157ce283ed360ea8a`. Development revisions keep +`switchyard-libsy`, `switchyard-protocol`, and `switchyard-translation` on the +same source revision to preserve their shared type identity. Revision pinning +is dependency maintenance, not a libsy feature gap; compatible Switchyard 0.2 +crate releases will replace the source pins. The current Relay draft follows +the focused stream-preservation work in +[NVIDIA-NeMo/Switchyard#192](https://github.com/NVIDIA-NeMo/Switchyard/pull/192). + +## Integration Contract Under Assessment + +```mermaid +flowchart LR + Caller["Caller wire request"] --> DecodeRequest["switchyard-translation decode"] + DecodeRequest --> Run["libsy run_stream"] + Run --> Decision["Decision"] + Decision --> Call["CallLlm"] + Call --> EncodeTarget["switchyard-translation encode target"] + EncodeTarget --> Dispatch["Relay provider dispatch"] + Dispatch --> DecodeResponse["switchyard-translation decode response or stream"] + DecodeResponse --> Respond["CallLlmRequest.respond"] + Respond --> Final["ReturnToAgent"] + Final --> EncodeCaller["switchyard-translation encode caller"] + EncodeCaller --> Caller +``` + +## Confirmed Baseline + +The pinned library supports the required random-router lifecycle: + +- client-less semantic targets; +- weighted and optionally seeded random selection; +- `Decision → CallLlm → ReturnToAgent` through `run_stream`; +- Relay-hosted buffered and streaming provider calls; +- concurrent independent runs; +- typed libsy orchestration and common provider-client errors; +- buffered request and response translation with exact same-format + preservation. + +## LIBSY-GAP-003: Exact Raw Stream-Event Preservation + +**Description:** A provider stream event must survive the +Relay → libsy → Relay round trip without losing fields that the neutral chunk +model does not understand. + +- **Priority:** P0 for same-protocol streaming parity. +- **Affected Relay behavior:** Same-protocol responses must preserve provider + extensions such as OpenAI Chat `system_fingerprint`. +- **Pinned behavior:** `switchyard-translation` decodes a raw event into + its normalized `LlmResponseChunk` values and later re-encodes those values. + Unknown fields disappear in this Switchyard translation round trip and + generated defaults can appear. +- **Reproducer:** Route an OpenAI Chat chunk containing a text delta and + `system_fingerprint` through the plugin. The pinned output drops the + fingerprint and adds `created: 0`. +- **Required behavior:** Carry the complete parsed source JSON beside its + normalized children. Replay that JSON when source and target formats match; + use only normalized children for cross-format encoding. +- **Proposed API area:** `switchyard-protocol::LlmResponseChunk` and + `switchyard-translation` stream decode/encode methods. +- **Upstream work:** `feat/translation-preserve-raw-stream-events`. +- **Switchyard issue/PR:** + [NVIDIA-NeMo/Switchyard#192](https://github.com/NVIDIA-NeMo/Switchyard/pull/192). + The Relay draft temporarily pins all three Switchyard crates to that PR's + commit. Updating the revisions after it lands is dependency maintenance, not + a separate libsy gap. +- **Acceptance:** JSON-semantically equivalent same-format events for OpenAI + Chat, OpenAI Responses, and Anthropic Messages; normalized cross-format text + translation; nested stream errors remain typed. +- **Relay test unblocked:** `same_protocol_stream_round_trip_preserves_raw_provider_event`. + +## LIBSY-GAP-004: Complete Provider Error Categories + +**Description:** libsy can represent transport, timeout, context-window, HTTP, +translation, and configuration failures, but the pinned protocol has no +dedicated model-unavailable category. + +- **Priority:** P2 for algorithms that react differently to model availability. +- **Affected Relay behavior:** Relay already has a typed + `ModelUnavailable` provider failure and uses the original Relay error for + retry classification. +- **Pinned behavior:** Relay can return the real failure message to libsy only + through the general client-error variant. +- **Reproducer:** Return Relay's typed `ModelUnavailable` failure from a + `CallLlm`. The host can retain that type for its own retry policy, but + `LlmClientError` can carry it back to the algorithm only as `General`. +- **Required behavior:** Add a typed model-unavailable variant without requiring + hosts to parse strings or invent an HTTP status. +- **Proposed API area:** `switchyard-protocol::LlmClientError`. +- **Switchyard issue/PR:** None; deferred until an algorithm needs to react to + this category. +- **Current integration impact:** Not a random-router blocker. The random router + does not branch on provider error category, and Relay retains the original + typed error for its retry decision. +- **Acceptance:** libsy receives a typed model-unavailable failure and existing + algorithms remain source-compatible. +- **Relay test unblocked:** A contract test proving that model-unavailable + reaches an algorithm as a distinct typed failure. + +## Deferred Capabilities + +Decision-only execution, observe-only routing, stage routing, ensemble-specific +concurrent dispatch, structured decision identity, confidence, baselines, and +optimization contributions are outside this integration. They are not mapped +around locally and do not block the random-router `run_stream` contract. diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md index ba50b355f..16b88e285 100644 --- a/crates/switchyard/README.md +++ b/crates/switchyard/README.md @@ -3,123 +3,76 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -[![License](https://img.shields.io/github/license/NVIDIA/NeMo-Relay)](https://github.com/NVIDIA/NeMo-Relay/blob/main/LICENSE) -[![GitHub](https://img.shields.io/badge/github-repo-blue?logo=github)](https://github.com/NVIDIA/NeMo-Relay/) -[![Release](https://img.shields.io/github/v/release/NVIDIA/NeMo-Relay?color=green)](https://github.com/NVIDIA/NeMo-Relay/releases) - # NeMo Relay Switchyard Plugin -`nemo-relay-switchyard` is NeMo Relay's experimental integration -with the [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) -Decision API. It adds routing-aware LLM execution intercepts to the Relay -runtime while preserving Relay ownership of provider credentials, target -bindings, dispatch, retries, fallbacks, and observability. - -NeMo Relay 0.6.0 uses a separately running Switchyard Decision API from the -`topic/nemo-relay-integration` branch. - -Install it from crates.io, or build it from the NeMo Relay source checkout with -the optional CLI feature while the Switchyard Decision API contract and -service/library boundary are still evolving. - -Use the plugin to: - -- **Route through Switchyard decisions**: Select an exact Relay-owned target - using a versioned Decision API contract. -- **Keep provider protocols stable**: Use Switchyard's translation library for - OpenAI Chat, OpenAI Responses, and Anthropic Messages request and response - translation. -- **Preserve Relay execution semantics**: Keep retries, trusted fallbacks, - credentials, streaming behavior, and optimization accounting in Relay. -- **Support staged rollout**: Run in enforce or observe-only mode with - explicit target bindings and protocol defaults. - -## Implementation and Runtime Behavior - -The plugin includes the following implementation and runtime behavior: - -- `SwitchyardConfig`: The typed plugin configuration contract. -- `SwitchyardRuntime`: Buffered and streaming routing intercepts. -- Decision and target validation for exact backend, model, protocol, and - endpoint bindings. -- ATOF-backed or payload-only routing context modes. -- Routing marks and model-routing optimization contributions for Relay's - cumulative accounting pipeline. -- Switchyard-owned protocol translation through the - `switchyard-translation` dependency. - -## Installation and Source Build - -Add the crate from crates.io: - -```bash -cargo add nemo-relay-switchyard -``` - -To build the optional CLI integration from a NeMo Relay source checkout: +`nemo-relay-switchyard` is NeMo Relay's experimental in-process integration +with [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard). +It runs libsy's random router inside Relay through `Algorithm::run_stream`. -```bash -cargo build -p nemo-relay-cli --features switchyard -cargo test -p nemo-relay-switchyard -``` +Relay owns provider target bindings, credentials, transport, retries, fallback, +and observability. libsy emits routing decisions and requests provider calls; +Relay fulfills each `CallLlm` and returns the real response or error through +`CallLlmRequest::respond`. No Switchyard server or Switchyard LLM client is +used. -The resulting CLI includes the Switchyard component only when the `switchyard` -feature is enabled. A default Relay build does not include this experimental -integration. +## Translation Boundary -## Runtime Boundary +`switchyard-translation` is the plugin's only provider wire-format translation +engine: -The current integration calls Switchyard's HTTP Decision API at runtime. Relay -does not start or supervise the Switchyard service. For ATOF-backed profiles, -Switchyard also provides the `/v1/atof/events` ingestion and accumulator -runtime. The service must therefore be running before Relay activates the -plugin. Activation performs a bounded request to the service's `/health` -endpoint and fails if it does not return `{"status":"ok"}`. This requirement -applies to enforce and observe-only rollout modes. +1. decode the caller request into the Switchyard neutral request; +2. encode each routed call for the selected target protocol; +3. decode the provider response or stream before returning it to libsy; +4. encode `ReturnToAgent` into the caller protocol. -The current service setup is documented in -[`examples/switchyard/README.md`](../../examples/switchyard/README.md), including -the pinned topic-branch commit, local configuration, compatibility smoke test, -and trajectory workflow. +Same-protocol buffered bodies and stream events preserve their original +provider JSON. Cross-protocol routes carry only data represented by the target +protocol and fail when Switchyard reports a lossy conversion. The plugin does +not use NeMo Relay codecs as a translation fallback. -Translation runs in-process through Switchyard's Rust translation library. - -## Configuration and Registration - -The CLI registers the component when built with `--features switchyard` and -accepts a `[[components]]` entry with `kind = "switchyard"`. A minimal -configuration selects the Decision API and trusted protocol defaults: +## Configuration ```toml +version = 1 + [[components]] kind = "switchyard" enabled = true [components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "my-profile" -context_mode = "payload_only" -request_materialization = "summary_only" +version = 2 +priority = 0 +max_retries = 3 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] -openai_chat = "my-openai-target" -openai_responses = "my-responses-target" -anthropic_messages = "my-anthropic-target" +openai_chat = "fast" + +[components.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 + +[components.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" ``` -For ATOF-backed profiles, configure an enabled Relay ATOF HTTP stream sink that -has a unique `name`, targets the Switchyard ingestion URL, and uses -environment-referenced authentication headers. Set `atof_endpoint_name` in the -Switchyard component to that name. Local ATOF JSONL output alone does not -populate the Switchyard accumulator. Keep provider and Decision API credentials -outside tracked configuration files. +Target-map keys are libsy semantic names. Relay configuration remains +authoritative for the physical model, protocol, endpoint, URL, and credentials. +Version-1 Decision API configuration is rejected with a migration error. -## Documentation +Build and test the optional CLI integration with: -For more information, refer to the following resources: +```bash +cargo build -p nemo-relay-cli --features switchyard +cargo test -p nemo-relay-switchyard +``` -- [Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about) -- [Switchyard configuration reference](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/configuration) -- [Switchyard integration examples](../../examples/switchyard/README.md) -- [Switchyard repository](https://github.com/NVIDIA-NeMo/Switchyard) +See [`examples/switchyard`](../../examples/switchyard) for a no-service, +fake-provider smoke test. diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs index e8d5fced8..0c3eb60c8 100644 --- a/crates/switchyard/src/component.rs +++ b/crates/switchyard/src/component.rs @@ -1,61 +1,54 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Switchyard plugin configuration and Relay execution integration. +//! Switchyard libsy plugin configuration and Relay execution integration. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; +use std::sync::{Arc, Mutex}; use futures_util::StreamExt; +use http::Uri; +use http::header::{HeaderName, HeaderValue}; use nemo_relay::api::event::{CategoryProfile, DataSchema, EventCategory}; use nemo_relay::api::llm::LlmRequest; -use nemo_relay::api::optimization::record_llm_optimization_contribution; use nemo_relay::api::runtime::{ - LlmExecutionFn, LlmJsonStream, LlmStreamExecutionFn, LlmStreamInner, + LlmExecutionFn, LlmExecutionNextFn, LlmJsonStream, LlmStreamExecutionFn, + LlmStreamExecutionNextFn, task_scope_top, }; use nemo_relay::api::scope::{EmitMarkEventParams, event}; -use nemo_relay::codec::optimization::{ - LlmOptimizationContribution, LlmOptimizationKind, LlmOptimizationModel, - LlmOptimizationModelTransition, -}; use nemo_relay::error::{FlowError, Result as FlowResult}; -use nemo_relay::observability::atof::{AtofEndpointFieldNamePolicy, AtofEndpointTransport}; use nemo_relay::plugin::{ - ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginConfig, PluginError, + ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginError, PluginRegistrationContext, Result as PluginResult, deregister_plugin, register_plugin, }; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json, json}; -use uuid::Uuid; +use switchyard_libsy::algorithms::Random; +use switchyard_libsy::{ + Algorithm, CallLlmRequest, Context as LibsyContext, Decision, LibsyError, LlmResponse, + LlmTarget, LlmTargetSet, Request as LibsyRequest, Response as LibsyResponse, Step, +}; +use switchyard_protocol::{LlmClientError, Metadata}; -use crate::contract::{ - DecisionAttempt, DecisionProfile, ROUTING_DECISION_SCHEMA_VERSION, - ROUTING_REQUEST_SCHEMA_VERSION, RequestIdentity, RequestMaterialization, RequestProtocol, - RequestSummary, RoutingDecision, RoutingRequest, RoutingTarget, +use crate::stream_translation::{ + StreamCloseTracker, flow_to_client_error, provider_response_stream, relay_response_stream, }; -use crate::stream_translation::StreamTranscoder; use crate::translation::{ - decode_request, encode_request, latest_user_prompt, recent_message_window, translate_response, - translation_engine, validate_portable_request, + decode_request, decode_response, encode_request, encode_response, translation_engine, + wire_format, }; /// Plugin kind used in Relay plugin configuration. pub const SWITCHYARD_PLUGIN_KIND: &str = "switchyard"; -const SWITCHYARD_HEALTH_PATH: &str = "/health"; -const SWITCHYARD_HEALTH_TIMEOUT: Duration = Duration::from_secs(2); -const SWITCHYARD_HEALTH_MAX_ATTEMPTS: usize = 3; -const SWITCHYARD_HEALTH_INITIAL_BACKOFF: Duration = Duration::from_millis(100); const INTERNAL_DISPATCH_URL_HEADER: &str = "x-nemo-relay-internal-dispatch-url"; const INTERNAL_DISPATCH_ROUTE_HEADER: &str = "x-nemo-relay-internal-dispatch-route"; const INTERNAL_RETRY_AWARE_HEADER: &str = "x-nemo-relay-internal-retry-aware"; +const RELAY_REQUEST_ID_HEADER: &str = "x-nemo-relay-request-id"; +const RELAY_TURN_ID_HEADER: &str = "x-nemo-relay-turn-id"; const ROUTING_MARK_SCHEMA: &str = "switchyard.routing_mark"; -const ROUTING_CONTRIBUTION_SCHEMA: &str = "nvidia.switchyard.routing_optimization"; /// Supported provider wire protocols. #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] @@ -71,7 +64,7 @@ pub enum WireProtocol { } impl WireProtocol { - fn label(self) -> &'static str { + pub(crate) fn label(self) -> &'static str { match self { Self::OpenaiChat => "openai_chat", Self::OpenaiResponses => "openai_responses", @@ -87,7 +80,7 @@ impl WireProtocol { } } - fn from_call(name: &str, request: &LlmRequest) -> Option { + fn from_call(name: &str) -> Option { match name { "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { Some(Self::OpenaiChat) @@ -96,68 +89,36 @@ impl WireProtocol { "anthropic.messages" | "anthropic" | "anthropic_messages" => { Some(Self::AnthropicMessages) } - _ if request.content.get("input").is_some() => Some(Self::OpenaiResponses), - _ if request.content.get("system").is_some() => Some(Self::AnthropicMessages), - _ if request.content.get("messages").is_some() => Some(Self::OpenaiChat), _ => None, } } } -/// Routing rollout mode. -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RoutingMode { - /// Apply Switchyard decisions. - #[default] - Enforce, - /// Record decisions but dispatch trusted defaults. - ObserveOnly, -} - -impl RoutingMode { - fn label(self) -> &'static str { - match self { - Self::Enforce => "enforce", - Self::ObserveOnly => "observe_only", - } - } -} - -/// Whether the selected Switchyard profile depends on ATOF-derived history. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum ContextMode { - /// The router uses only current request material. - PayloadOnly, - /// Stable identity and a configured ATOF endpoint are required. - AtofRequired, -} - -/// Exact Relay-owned backend binding for one Switchyard backend ID. +/// Exact Relay-owned provider binding for one libsy semantic target. #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct TargetBinding { - /// Exact model expected in the Switchyard decision. + /// Provider model sent on the physical request. pub model: String, - /// Exact protocol expected in the Switchyard decision. + /// Provider wire protocol. pub protocol: WireProtocol, - /// Exact endpoint expected in the Switchyard decision. + /// Provider endpoint. pub endpoint: String, - /// Relay-owned backend base URL. + /// Relay-owned provider base URL. pub base_url: String, - /// Static non-sensitive backend headers. + /// Relative random-routing weight. + #[serde(default = "default_weight")] + pub weight: f64, + /// Static non-sensitive provider headers. #[serde(default)] pub headers: BTreeMap, - /// Backend headers resolved from environment variables. + /// Provider headers resolved from environment variables. #[serde(default)] pub header_env: BTreeMap, } -/// Trusted fallback target IDs for each inbound protocol. -#[derive(Clone, Debug, Deserialize, Serialize)] +/// Trusted fallback target names for each inbound protocol. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct ProtocolDefaults { /// OpenAI Chat fallback target. @@ -181,111 +142,81 @@ impl ProtocolDefaults { } } +/// In-process libsy algorithm configuration. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AlgorithmConfig { + /// Uniform or weighted random routing. + Random { + /// Optional deterministic random seed. + #[serde(default)] + seed: Option, + }, +} + +impl Default for AlgorithmConfig { + fn default() -> Self { + Self::Random { seed: None } + } +} + /// Versioned Switchyard plugin configuration. #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct SwitchyardConfig { - /// Config schema version. + /// Config schema version. The libsy-only contract is version 2. #[serde(default = "default_version")] pub version: u32, - /// Enforce or observe-only rollout mode. - #[serde(default)] - pub mode: RoutingMode, /// Execution-intercept priority. #[serde(default)] pub priority: i32, - /// Switchyard Decision API URL. - pub decision_api_url: String, - /// Switchyard profile ID. - pub decision_profile_id: String, - /// Current-request materialization. - pub request_materialization: RequestMaterialization, - /// Profile context requirement. - pub context_mode: ContextMode, - /// Decision call timeout. - #[serde(default = "default_decision_timeout_millis")] - pub decision_timeout_millis: u64, - /// Provider retries after the initial attempt. + /// Provider retries after the initial libsy run. #[serde(default = "default_max_retries")] pub max_retries: u32, - /// Number of messages in recent-message materialization. - #[serde(default = "default_recent_message_count")] - pub recent_message_count: usize, - /// Static non-sensitive Decision API headers. - #[serde(default)] - pub decision_headers: BTreeMap, - /// Decision API headers resolved from environment variables. + /// In-process libsy algorithm. #[serde(default)] - pub decision_header_env: BTreeMap, - /// Enabled inbound protocols. - #[serde(default = "default_enabled_protocols")] - pub enabled_inbound_profiles: BTreeSet, - /// Exact backend bindings keyed by Switchyard backend ID. + pub algorithm: AlgorithmConfig, + /// Semantic libsy targets mapped to Relay-owned provider bindings. pub targets: BTreeMap, /// Trusted per-protocol fallbacks. - pub default_targets: ProtocolDefaults, - /// Named observability ATOF endpoint used by history-backed profiles. #[serde(default)] - pub atof_endpoint_name: Option, + pub default_targets: ProtocolDefaults, + /// Inbound protocols handled by this component. + #[serde(default = "default_enabled_protocols")] + pub enabled_inbound_profiles: BTreeSet, } impl Default for SwitchyardConfig { fn default() -> Self { Self { version: default_version(), - mode: RoutingMode::default(), priority: 0, - decision_api_url: "http://127.0.0.1:8080/v1/routing/decision".into(), - decision_profile_id: String::new(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: default_decision_timeout_millis(), max_retries: default_max_retries(), - recent_message_count: default_recent_message_count(), - decision_headers: BTreeMap::new(), - decision_header_env: BTreeMap::new(), - enabled_inbound_profiles: default_enabled_protocols(), + algorithm: AlgorithmConfig::default(), targets: BTreeMap::new(), - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: String::new(), - anthropic_messages: String::new(), - }, - atof_endpoint_name: None, + default_targets: ProtocolDefaults::default(), + enabled_inbound_profiles: default_enabled_protocols(), } } } nemo_relay::editor_config! { impl SwitchyardConfig { - mode => { label: "Rollout mode", kind: Enum, values: ["enforce", "observe_only"] }, priority => { label: "Intercept priority", kind: Integer }, - decision_api_url => { label: "Decision API URL", kind: String }, - decision_profile_id => { label: "Decision profile ID", kind: String }, - request_materialization => { - label: "Request materialization", - kind: Enum, - values: ["none", "summary_only", "latest_user_prompt", "recent_message_window", "annotated_request", "full_body"] - }, - context_mode => { label: "Context mode", kind: Enum, values: ["payload_only", "atof_required"] }, - decision_timeout_millis => { label: "Decision timeout (ms)", kind: Integer }, max_retries => { label: "Maximum provider retries", kind: Integer }, - recent_message_count => { label: "Recent message count", kind: Integer }, - decision_headers => { label: "Decision API static headers", kind: StringMap }, - decision_header_env => { label: "Decision API environment headers", kind: StringMap }, - enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json }, - targets => { label: "Backend target bindings", kind: Json }, + algorithm => { label: "libsy algorithm", kind: Json }, + targets => { label: "Provider target bindings", kind: Json }, default_targets => { label: "Trusted protocol defaults", kind: Json }, - atof_endpoint_name => { label: "ATOF endpoint name", kind: String, optional: true } + enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json } } } impl From for PluginComponentSpec { fn from(value: SwitchyardConfig) -> Self { - let Json::Object(config) = - serde_json::to_value(value).expect("Switchyard config should serialize to an object") - else { - unreachable!("Switchyard config must serialize to an object") + let config = match serde_json::to_value(value) { + Ok(Json::Object(config)) => config, + _ => Map::new(), }; Self { kind: SWITCHYARD_PLUGIN_KIND.into(), @@ -296,17 +227,17 @@ impl From for PluginComponentSpec { } fn default_version() -> u32 { - 1 -} -fn default_decision_timeout_millis() -> u64 { - 25 + 2 } + fn default_max_retries() -> u32 { 3 } -fn default_recent_message_count() -> usize { - 8 + +fn default_weight() -> f64 { + 1.0 } + fn default_enabled_protocols() -> BTreeSet { BTreeSet::from([ WireProtocol::OpenaiChat, @@ -351,10 +282,7 @@ impl Plugin for SwitchyardPlugin { .and_then(SwitchyardRuntime::new) .map_err(PluginError::InvalidConfig)?, ); - runtime - .require_healthy_sidecar() - .await - .map_err(PluginError::RegistrationFailed)?; + let buffered = Arc::clone(&runtime); let buffered_intercept: LlmExecutionFn = Arc::new(move |name, request, next| { let runtime = Arc::clone(&buffered); @@ -362,7 +290,7 @@ impl Plugin for SwitchyardPlugin { Box::pin(async move { runtime.execute_buffered(&name, request, next).await }) }); ctx.register_llm_execution_intercept( - "decision", + "libsy", runtime.config.priority, buffered_intercept, )?; @@ -374,7 +302,7 @@ impl Plugin for SwitchyardPlugin { Box::pin(async move { runtime.execute_stream(&name, request, next).await }) }); ctx.register_llm_stream_execution_intercept( - "decision_stream", + "libsy_stream", runtime.config.priority, stream_intercept, )?; @@ -399,939 +327,556 @@ pub fn deregister_switchyard_component() -> bool { deregister_plugin(SWITCHYARD_PLUGIN_KIND) } -/// Validate the cross-component ATOF requirement for enabled history-backed profiles. -pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<(), String> { - let Some(component) = config - .components - .iter() - .find(|component| component.enabled && component.kind == SWITCHYARD_PLUGIN_KIND) - else { - return Ok(()); - }; - let switchyard = parse_config(&component.config)?; - if switchyard.context_mode != ContextMode::AtofRequired { - return Ok(()); - } - let required_name = validate_atof_endpoint_name(switchyard.atof_endpoint_name.as_deref())? - .ok_or_else(|| { - "atof_required Switchyard profiles require atof_endpoint_name".to_string() - })?; - let observability = config - .components - .iter() - .find(|component| component.enabled && component.kind == "observability") - .ok_or_else(|| "atof_required Switchyard profiles require observability".to_string())?; - let sinks = observability - .config - .get("atof") - .filter(|atof| atof.get("enabled").and_then(Json::as_bool) == Some(true)) - .and_then(|atof| atof.get("sinks")) - .and_then(Json::as_array) - .ok_or_else(|| { - "atof_required Switchyard profiles require an enabled ATOF endpoint".to_string() - })?; - let matching_sinks = sinks - .iter() - .filter(|sink| { - sink.get("type").and_then(Json::as_str) == Some("stream") - && sink.get("name").and_then(Json::as_str) == Some(required_name) - }) - .collect::>(); - let endpoint = match matching_sinks.as_slice() { - [sink] => *sink, - [] => { - return Err(format!( - "atof_required Switchyard profile requires named ATOF endpoint {required_name:?}" - )); - } - _ => { - return Err(format!( - "ATOF endpoint name {required_name:?} must resolve to exactly one endpoint" - )); - } - }; - let transport = endpoint.get("transport").map_or_else( - || Some(AtofEndpointTransport::default()), - |value| value.as_str().and_then(AtofEndpointTransport::parse), - ); - if transport != Some(AtofEndpointTransport::HttpPost) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use transport = http_post" - )); - } - let field_name_policy = endpoint.get("field_name_policy").map_or_else( - || Some(AtofEndpointFieldNamePolicy::default()), - |value| value.as_str().and_then(AtofEndpointFieldNamePolicy::parse), - ); - if field_name_policy != Some(AtofEndpointFieldNamePolicy::Preserve) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use field_name_policy = preserve" - )); - } - if endpoint - .get("header_env") - .and_then(Json::as_object) - .is_none_or(Map::is_empty) +fn parse_config(config: &Map) -> Result { + if config.get("version").and_then(Json::as_u64) == Some(1) + || config.contains_key("decision_api_url") { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} authentication must use at least one environment-referenced header" - )); + return Err( + "Switchyard config version 1 used the removed switchyard-server Decision API; migrate to version = 2 with [components.config.algorithm]".into(), + ); } - Ok(()) -} - -fn parse_config(config: &Map) -> Result { serde_json::from_value(Json::Object(config.clone())) .map_err(|error| format!("invalid Switchyard plugin config: {error}")) } struct SwitchyardRuntime { config: SwitchyardConfig, - client: reqwest::Client, + algorithm: Arc, target_headers: BTreeMap>, translation: switchyard_translation::TranslationEngine, } -enum BufferedAttempt { - Complete(Json), - Retry((String, String)), - Fallback(&'static str), -} - -enum StreamAttempt { - Committed(LlmJsonStream), - Retry((String, String)), - Fallback(&'static str), -} - -struct StreamAttemptContext { - routing_request: RoutingRequest, - decision: RoutingDecision, - attempt: u32, - max_attempts: u32, -} - -fn provider_fallback_reason(error: &FlowError) -> &'static str { - if error_is_retryable(error) { - "retry_exhausted" - } else { - "non_retryable_provider_error" - } -} - impl SwitchyardRuntime { fn new(config: SwitchyardConfig) -> Result { validate_config(&config)?; - let headers = resolve_headers(&config.decision_headers, &config.decision_header_env)?; - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(Duration::from_millis(config.decision_timeout_millis)) - .build() - .map_err(|error| format!("failed to build Decision API client: {error}"))?; let target_headers = config .targets .iter() - .map(|(id, target)| { - let headers = resolve_json_headers(&target.headers, &target.header_env)?; - Ok((id.clone(), headers)) + .map(|(name, target)| { + resolve_json_headers(&target.headers, &target.header_env) + .map(|headers| (name.clone(), headers)) }) - .collect::>()?; + .collect::>()?; + let algorithm = build_algorithm(&config)?; Ok(Self { config, - client, + algorithm, target_headers, translation: translation_engine(), }) } - // Skip the portability guard when no configured target uses a different protocol: with no - // possible cross-protocol translation, provider-specific fields never need to be portable. - fn may_translate_protocol(&self, inbound: WireProtocol) -> bool { - self.config - .targets - .values() - .any(|target| target.protocol != inbound) - } - - async fn require_healthy_sidecar(&self) -> Result<(), String> { - let health_url = switchyard_health_url(&self.config.decision_api_url)?; - let client = reqwest::Client::builder() - .timeout(SWITCHYARD_HEALTH_TIMEOUT) - .build() - .map_err(|error| format!("failed to build Switchyard health client: {error}"))?; - let mut backoff = SWITCHYARD_HEALTH_INITIAL_BACKOFF; - let mut final_error = None; - for attempt in 1..=SWITCHYARD_HEALTH_MAX_ATTEMPTS { - match check_switchyard_health(&client, &health_url).await { - Ok(()) => return Ok(()), - Err(error) => final_error = Some(error), - } - if attempt < SWITCHYARD_HEALTH_MAX_ATTEMPTS { - tokio::time::sleep(backoff).await; - backoff *= 2; - } - } - Err(final_error.expect("at least one Switchyard health attempt is configured")) - } - async fn execute_buffered( &self, name: &str, original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, + next: LlmExecutionNextFn, ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { + let Some(inbound) = WireProtocol::from_call(name) else { return next(original).await; }; if !self.config.enabled_inbound_profiles.contains(&inbound) { return next(original).await; } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_buffered( - inbound, - original, - next, - "unsupported_provider_extension", - ) - .await; - } - - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_buffered(inbound, original, next, "observe_only") - .await; - } let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; for attempt in 1..=max_attempts { - match self - .buffered_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - BufferedAttempt::Complete(response) => return Ok(response), - BufferedAttempt::Retry(retry) => previous = Some(retry), - BufferedAttempt::Fallback(reason) => { + self.emit_requested(&original, inbound, attempt); + let request = match self.libsy_request(inbound, &original, false) { + Ok(request) => request, + Err(error) => { + self.emit_error( + attempt, + "request_translation", + &error.to_string(), + &original, + ); return self - .dispatch_fallback_buffered(inbound, original, next, reason) + .dispatch_fallback_buffered(inbound, original, next, "translation_error") .await; } - } - } - unreachable!("routing attempt loop always returns") - } - - async fn buffered_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(BufferedAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - match next(routed).await { - Ok(response) => { - match translate_response(&self.translation, target_protocol, inbound, &response) { - Ok(response) => { - self.record_routing_contribution(&decision, attempt, true); - Ok(BufferedAttempt::Complete(response)) - } + }; + match self + .drive_buffered_run( + request, + dispatch_headers(&original.headers), + next.clone(), + attempt, + ) + .await + { + Ok(response) => match self.finish_buffered(inbound, response) { + Ok(response) => return Ok(response), Err(error) => { self.emit_error( - Some(&routing_request), attempt, "response_translation", &error.to_string(), + &original, ); - Ok(BufferedAttempt::Fallback("translation_error")) + return self + .dispatch_fallback_buffered( + inbound, + original, + next, + "translation_error", + ) + .await; } + }, + Err(failure) if failure.is_retryable() && attempt < max_attempts => { + self.emit_retry(attempt, &failure.error.to_string(), &original); + } + Err(failure) => { + let reason = if failure.is_retryable() { + "retry_exhausted" + } else { + "non_retryable_libsy_error" + }; + self.emit_error(attempt, "libsy_run", &failure.error.to_string(), &original); + return self + .dispatch_fallback_buffered(inbound, original, next, reason) + .await; } - } - Err(error) if error_is_retryable(&error) && attempt < max_attempts => { - let retry_reason = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &retry_reason); - self.emit_retry(&routing_request, &decision, attempt, &retry_reason); - Ok(BufferedAttempt::Retry(( - decision.route.backend_id, - retry_reason, - ))) - } - Err(error) => { - let summary = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &summary); - Ok(BufferedAttempt::Fallback(provider_fallback_reason(&error))) } } + Err(FlowError::Internal( + "Switchyard retry loop ended without a result".into(), + )) } async fn execute_stream( &self, name: &str, original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, + next: LlmStreamExecutionNextFn, ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { + let Some(inbound) = WireProtocol::from_call(name) else { return next(original).await; }; if !self.config.enabled_inbound_profiles.contains(&inbound) { return next(original).await; } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_stream(inbound, original, next, "unsupported_provider_extension") - .await; - } - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_stream(inbound, original, next, "observe_only") - .await; - } let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; for attempt in 1..=max_attempts { + self.emit_requested(&original, inbound, attempt); + let request = match self.libsy_request(inbound, &original, true) { + Ok(request) => request, + Err(error) => { + self.emit_error( + attempt, + "request_translation", + &error.to_string(), + &original, + ); + return self + .dispatch_fallback_stream(inbound, original, next, "translation_error") + .await; + } + }; + let tracker = StreamCloseTracker::default(); match self - .stream_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? + .drive_stream_run( + request, + dispatch_headers(&original.headers), + next.clone(), + tracker.clone(), + attempt, + ) + .await { - StreamAttempt::Committed(stream) => return Ok(stream), - StreamAttempt::Retry(retry) => previous = Some(retry), - StreamAttempt::Fallback(reason) => { + Ok(response) => match self.finish_stream(inbound, response, tracker) { + Ok(response) => return Ok(response), + Err(error) => { + self.emit_error( + attempt, + "response_translation", + &error.to_string(), + &original, + ); + return self + .dispatch_fallback_stream(inbound, original, next, "translation_error") + .await; + } + }, + Err(failure) if failure.is_retryable() && attempt < max_attempts => { + self.emit_retry(attempt, &failure.error.to_string(), &original); + } + Err(failure) => { + let reason = if failure.is_retryable() { + "retry_exhausted" + } else { + "non_retryable_libsy_error" + }; + self.emit_error( + attempt, + "libsy_stream_run", + &failure.error.to_string(), + &original, + ); return self .dispatch_fallback_stream(inbound, original, next, reason) .await; } } } - unreachable!("stream routing attempt loop always returns") + Err(FlowError::Internal( + "Switchyard stream retry loop ended without a result".into(), + )) } - async fn stream_attempt( + async fn drive_buffered_run( &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmStreamExecutionNextFn, + request: LibsyRequest, + headers: Map, + next: LlmExecutionNextFn, attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(StreamAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - let context = StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - }; - match next(routed).await { - Ok(mut upstream) => { - let first = upstream.next().await; - Ok(self.classify_open_stream(inbound, target_protocol, context, upstream, first)) + ) -> Result { + let values = context_values(request.metadata.as_ref(), attempt); + let mark_metadata = libsy_identity_metadata(request.metadata.as_ref()); + let mut context = LibsyContext::default(); + context.values = values.into_iter().collect(); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); + } + Ok(Step::CallLlm(call)) => { + self.serve_buffered_call( + *call, + headers.clone(), + next.clone(), + Arc::clone(&provider_error), + ) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(RunFailure::new(error, &provider_error)), } - Err(error) => Ok(self.classify_stream_setup_error(context, error)), } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) } - fn classify_open_stream( + async fn drive_stream_run( &self, - inbound: WireProtocol, - target_protocol: WireProtocol, - context: StreamAttemptContext, - upstream: LlmJsonStream, - first: Option>, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - match first { - Some(Ok(first)) => { - self.record_routing_contribution(&decision, attempt, true); - let committed = LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(first)), - upstream, - }); - let output = if target_protocol == inbound { - committed - } else { - translated_stream( - target_protocol, - inbound, - decision.route.target_model.clone(), - committed, + request: LibsyRequest, + headers: Map, + next: LlmStreamExecutionNextFn, + tracker: StreamCloseTracker, + attempt: u32, + ) -> Result { + let values = context_values(request.metadata.as_ref(), attempt); + let mark_metadata = libsy_identity_metadata(request.metadata.as_ref()); + let mut context = LibsyContext::default(); + context.values = values.into_iter().collect(); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); + } + Ok(Step::CallLlm(call)) => { + self.serve_stream_call( + *call, + headers.clone(), + next.clone(), + tracker.clone(), + Arc::clone(&provider_error), ) - }; - StreamAttempt::Committed(mark_terminal_stream( - output, - "provider_stream_committed", - self.config.mode.label(), - identity_metadata(&routing_request), - )) - } - Some(Err(error)) if error_is_retryable(&error) && attempt < max_attempts => self - .retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - provider_error_summary(&error), - ), - None if attempt < max_attempts => self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - "empty_stream".into(), - ), - Some(Err(error)) => { - let summary = provider_error_summary(&error); - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_open", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(RunFailure::new(error, &provider_error)), } - None => StreamAttempt::Fallback("empty_stream"), } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) } - fn classify_stream_setup_error( + async fn serve_buffered_call( &self, - context: StreamAttemptContext, - error: FlowError, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - let summary = provider_error_summary(&error); - if error_is_retryable(&error) && attempt < max_attempts { - return self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_setup", - summary, - ); + call: CallLlmRequest, + headers: Map, + next: LlmExecutionNextFn, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let result = async { + let metadata = routed.request.metadata.clone(); + let (target_protocol, request) = + self.apply_target(&target_name, routed.request, headers, false)?; + let response = match next(request).await { + Ok(response) => response, + Err(error) => { + remember_provider_error(&provider_error, &error); + return Err(flow_to_client_error(error, &target_name)); + } + }; + let response = decode_response(&self.translation, target_protocol, &response) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + Ok(LibsyResponse { + llm_response: LlmResponse::Agg(response), + metadata: Some(response_metadata(metadata.as_ref(), target_protocol)), + }) } - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_setup", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) } - fn retry_stream_attempt( + async fn serve_stream_call( &self, - routing_request: &RoutingRequest, - decision: RoutingDecision, - attempt: u32, - error_class: &str, - reason: String, - ) -> StreamAttempt { - if reason != "empty_stream" { - self.emit_error(Some(routing_request), attempt, error_class, &reason); + call: CallLlmRequest, + headers: Map, + next: LlmStreamExecutionNextFn, + tracker: StreamCloseTracker, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let result = async { + let metadata = routed.request.metadata.clone(); + let (target_protocol, request) = + self.apply_target(&target_name, routed.request, headers, true)?; + let mut upstream = match next(request).await { + Ok(upstream) => upstream, + Err(error) => { + remember_provider_error(&provider_error, &error); + return Err(flow_to_client_error(error, &target_name)); + } + }; + let first = match upstream.next().await { + Some(Ok(first)) => first, + Some(Err(error)) => { + remember_provider_error(&provider_error, &error); + return Err(flow_to_client_error(error, &target_name)); + } + None => { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }); + } + }; + let response = provider_response_stream( + upstream, + wire_format(target_protocol), + first, + tracker, + target_name.clone(), + )?; + Ok(LibsyResponse { + llm_response: LlmResponse::Stream(response), + metadata: Some(response_metadata(metadata.as_ref(), target_protocol)), + }) } - self.emit_retry(routing_request, &decision, attempt, &reason); - StreamAttempt::Retry((decision.route.backend_id, reason)) + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) } - async fn decided_request( + fn libsy_request( &self, inbound: WireProtocol, original: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result<(RoutingRequest, RoutingDecision, LlmRequest), String> { - let request = self.routing_request(inbound, original, attempt, previous)?; - self.emit_requested(&request); - let started = Instant::now(); - let response = self - .client - .post(&self.config.decision_api_url) - .header("x-nemo-relay-session-id", &request.identity.session_id) - .json(&request) - .send() - .await - .map_err(|error| format!("Decision API request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format!("Decision API returned HTTP {status}: {body}")); - } - let decision = response - .json::() - .await - .map_err(|error| format!("Decision API returned invalid JSON: {error}"))?; - self.validate_decision(&decision)?; - if let Some(baseline) = decision.baseline_route.as_ref() - && let Err(error) = self.validate_target(baseline) - { - self.emit_error(Some(&request), attempt, "baseline_binding", &error); - } - let routed = self.apply_target(inbound, original.clone(), &decision)?; - let latency = started.elapsed().as_millis() as u64; - self.emit_decision( - &request, - &decision, - attempt, - self.config.mode == RoutingMode::ObserveOnly, - latency, - ); - Ok((request, decision, routed)) - } - - fn routing_request( - &self, - inbound: WireProtocol, - request: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result { - let session = header(request, "x-nemo-relay-session-id"); - let stable_request_id = header(request, "x-nemo-relay-request-id"); - if self.config.context_mode == ContextMode::AtofRequired - && (session.is_none() || stable_request_id.is_none()) - { - return Err("stable session and request identity are required for this profile".into()); - } - let identity_is_stable = session.is_some() && stable_request_id.is_some(); - let synthetic_session = format!("request-{}", Uuid::now_v7()); - let session_id = session.unwrap_or_else(|| synthetic_session.clone()); - let request_id = stable_request_id.unwrap_or_else(|| format!("request-{}", Uuid::now_v7())); - let annotated = decode_request(&self.translation, inbound, request) - .map_err(|error| format!("request translation decode failed: {error}"))?; - let current_request = self.materialize(inbound, request, &annotated)?; - let (previous_route, retry_reason) = previous.unzip(); - Ok(RoutingRequest { - schema_version: ROUTING_REQUEST_SCHEMA_VERSION.into(), - decision_profile: DecisionProfile { - profile_id: self.config.decision_profile_id.clone(), - request_materialization: self.config.request_materialization, - }, - identity: RequestIdentity { - session_id, - request_id, - turn_id: header(request, "x-nemo-relay-turn-id"), - parent_scope_id: header(request, "x-nemo-relay-parent-scope-id"), - root_scope_id: header(request, "x-nemo-relay-root-scope-id"), - harness: header(request, "x-nemo-relay-agent-kind") - .unwrap_or_else(|| "unknown".into()), - source: header(request, "x-nemo-relay-source") - .unwrap_or_else(|| "nemo-relay".into()), - owner_id: header(request, "x-nemo-relay-owner-id"), - quality: header(request, "x-nemo-relay-identity-quality").unwrap_or_else(|| { - if identity_is_stable { - "explicit".into() - } else { - "synthetic".into() - } - }), - }, - protocol: RequestProtocol { - inbound_profile: inbound.label().into(), - inbound_endpoint: inbound.endpoint().into(), - desired_response_profile: inbound.label().into(), - }, - request_summary: RequestSummary { - client_requested_model: request - .content - .get("model") - .and_then(Json::as_str) - .map(ToOwned::to_owned), - prompt_token_estimate: None, - tool_count_in_payload: request - .content - .get("tools") - .and_then(Json::as_array) - .map(|tools| tools.len() as u64), - has_system_prompt: Some( - annotated.instructions.iter().any(|instruction| { - instruction.role == switchyard_translation::Role::System - }) || annotated - .messages - .iter() - .any(|message| message.role == switchyard_translation::Role::System), - ), - }, - current_request, - attempt: DecisionAttempt { - routing_attempt: attempt, - max_routing_attempts: self.config.max_retries.saturating_add(1), - previous_route, - retry_reason, - }, + streaming: bool, + ) -> FlowResult { + let mut llm_request = decode_request(&self.translation, inbound, original)?; + llm_request.stream = streaming; + let mut metadata = relay_metadata(&string_headers(&original.headers)); + metadata.wire_format = Some(wire_format(inbound)); + Ok(LibsyRequest { + llm_request, + raw_request: Some(original.content.clone()), + metadata: Some(metadata), }) } - fn materialize( - &self, - inbound: WireProtocol, - request: &LlmRequest, - annotated: &switchyard_translation::LlmRequest, - ) -> Result, String> { - match self.config.request_materialization { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => Ok(None), - RequestMaterialization::FullBody => Ok(Some(json!({"body": request.content}))), - RequestMaterialization::AnnotatedRequest => Ok(Some(json!({ - "body": request.content, - "annotated_request": annotated, - }))), - RequestMaterialization::LatestUserPrompt => { - let prompt = latest_user_prompt(annotated) - .ok_or_else(|| "latest_user_prompt requires a user message".to_string())?; - let latest = recent_message_window(annotated, 1); - let body = encode_request(&self.translation, inbound, &latest, Map::new()) - .map_err(|error| format!("latest user prompt encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "latest_user_prompt": prompt}))) - } - RequestMaterialization::RecentMessageWindow => { - let window = recent_message_window(annotated, self.config.recent_message_count); - let body = encode_request(&self.translation, inbound, &window, Map::new()) - .map_err(|error| format!("recent window encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "annotated_request": window}))) - } - } - } - - fn validate_decision(&self, decision: &RoutingDecision) -> Result<(), String> { - if decision.schema_version != ROUTING_DECISION_SCHEMA_VERSION { - return Err(format!( - "unsupported decision schema {:?}", - decision.schema_version - )); - } - self.validate_target(&decision.route).map(|_| ()) - } - - fn validate_target(&self, target: &RoutingTarget) -> Result<&TargetBinding, String> { - let binding = self - .config - .targets - .get(&target.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", target.backend_id))?; - if binding.model != target.target_model - || binding.protocol.label() != target.target_protocol_profile - || binding.endpoint != target.target_endpoint - { - return Err(format!( - "decision target {:?} does not match its exact Relay binding", - target.backend_id - )); - } - Ok(binding) - } - - fn record_routing_contribution(&self, decision: &RoutingDecision, attempt: u32, applied: bool) { - let Some(contribution) = self.routing_contribution(decision, attempt, applied) else { - return; - }; - let _ = record_llm_optimization_contribution(contribution); - } - - fn routing_contribution( - &self, - decision: &RoutingDecision, - attempt: u32, - applied: bool, - ) -> Option { - let baseline = decision - .baseline_route - .as_ref() - .filter(|baseline| self.validate_target(baseline).is_ok())?; - let mut contribution = LlmOptimizationContribution::new( - SWITCHYARD_PLUGIN_KIND, - LlmOptimizationKind::model_routing(), - ); - contribution.applied = applied; - contribution.model_transition = Some(LlmOptimizationModelTransition { - baseline: Some(LlmOptimizationModel::new(&baseline.target_model)), - effective: Some(LlmOptimizationModel::new(&decision.route.target_model)), - }); - contribution.payload_schema = Some(DataSchema { - name: ROUTING_CONTRIBUTION_SCHEMA.to_string(), - version: "1".to_string(), - }); - contribution.payload = Some(json!({ - "decision_id": decision.decision_id, - "selected_backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "baseline_backend_id": baseline.backend_id, - "baseline_tier": baseline.tier, - "routing_attempt": attempt, - "rollout_mode": self.config.mode.label(), - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - })); - Some(contribution) - } - fn apply_target( &self, - inbound: WireProtocol, - request: LlmRequest, - decision: &RoutingDecision, - ) -> Result { - let binding = self - .config - .targets - .get(&decision.route.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", decision.route.backend_id))?; - let annotated = decode_request(&self.translation, inbound, &request) - .map_err(|error| format!("request decode failed: {error}"))?; - let mut routed = if inbound == binding.protocol { - request - } else { - encode_request( - &self.translation, - binding.protocol, - &annotated, - request.headers, - ) - .map_err(|error| format!("request translation failed: {error}"))? - }; - let object = routed - .content - .as_object_mut() - .ok_or_else(|| "translated request body is not an object".to_string())?; - object.insert("model".into(), Json::String(binding.model.clone())); - if let Some(headers) = self.target_headers.get(&decision.route.backend_id) { - routed.headers.extend(headers.clone()); + target_name: &str, + mut request: LibsyRequest, + headers: Map, + streaming: bool, + ) -> Result<(WireProtocol, LlmRequest), LlmClientError> { + let target = + self.config + .targets + .get(target_name) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("libsy selected unknown target {target_name:?}"), + })?; + request.llm_request.stream = streaming; + let mut routed = encode_request( + &self.translation, + target.protocol, + &request.llm_request, + headers, + ) + .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; + let object = routed.content.as_object_mut().ok_or_else(|| { + LlmClientError::RequestEncoding("translated provider request is not an object".into()) + })?; + object.insert("model".into(), Json::String(target.model.clone())); + object.insert("stream".into(), Json::Bool(streaming)); + if let Some(target_headers) = self.target_headers.get(target_name) { + routed.headers.extend(target_headers.clone()); } routed.headers.insert( INTERNAL_DISPATCH_ROUTE_HEADER.into(), - Json::String(binding.protocol.label().into()), + Json::String(target.protocol.label().into()), ); routed.headers.insert( INTERNAL_DISPATCH_URL_HEADER.into(), - Json::String(dispatch_url(&binding.base_url, &binding.endpoint)), + Json::String(dispatch_url(&target.base_url, &target.endpoint)), ); routed.headers.insert( INTERNAL_RETRY_AWARE_HEADER.into(), Json::String("true".into()), ); - Ok(routed) + Ok((target.protocol, routed)) } - fn fallback_request( + fn finish_buffered(&self, inbound: WireProtocol, response: LibsyResponse) -> FlowResult { + match response.llm_response { + LlmResponse::Agg(response) => encode_response(&self.translation, inbound, &response), + LlmResponse::Stream(_) => Err(FlowError::Internal( + "libsy returned a stream for a buffered Relay request".into(), + )), + } + } + + fn finish_stream( &self, inbound: WireProtocol, - request: LlmRequest, - ) -> Result { - let id = self.config.default_targets.target(inbound); - let binding = self - .config - .targets - .get(id) - .ok_or_else(|| format!("unknown fallback target {id:?}"))?; - let decision = RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "relay-fallback".into(), - router: crate::contract::DecisionProvider { - name: "relay-fallback".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "fallback".into(), - target_model: binding.model.clone(), - backend_id: id.to_string(), - target_protocol_profile: binding.protocol.label().into(), - target_endpoint: binding.endpoint.clone(), - }, - baseline_route: None, - confidence: None, - reason_code: Some("relay_trusted_fallback".into()), - reason_summary: None, - metadata: BTreeMap::new(), - extra: BTreeMap::new(), - }; - self.apply_target(inbound, request, &decision) + response: LibsyResponse, + tracker: StreamCloseTracker, + ) -> FlowResult { + let source = response + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format) + .ok_or_else(|| { + FlowError::Internal("libsy streaming response omitted its source format".into()) + })?; + match response.llm_response { + LlmResponse::Stream(response) => Ok(relay_response_stream( + response, + source, + wire_format(inbound), + tracker, + )), + LlmResponse::Agg(_) => Err(FlowError::Internal( + "libsy returned a buffered aggregate for a streaming Relay request".into(), + )), + } } async fn dispatch_fallback_buffered( &self, inbound: WireProtocol, original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, + next: LlmExecutionNextFn, reason: &str, ) -> FlowResult { self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(response) => Ok(response), - Err(error) => { - emit_terminal_error( - &error, - "fallback_buffered", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } + let request = self.fallback_request(inbound, original, false)?; + next(request).await } async fn dispatch_fallback_stream( &self, inbound: WireProtocol, original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, + next: LlmStreamExecutionNextFn, reason: &str, ) -> FlowResult { self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(stream) => Ok(mark_terminal_stream( - stream, - "fallback_stream", - self.config.mode.label(), - metadata.clone(), - )), - Err(error) => { - emit_terminal_error( - &error, - "fallback_stream_setup", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } + let request = self.fallback_request(inbound, original, true)?; + next(request).await } - fn emit_requested(&self, request: &RoutingRequest) { + fn fallback_request( + &self, + inbound: WireProtocol, + original: LlmRequest, + streaming: bool, + ) -> FlowResult { + let target = self.config.default_targets.target(inbound); + let request = self.libsy_request(inbound, &original, streaming)?; + self.apply_target( + target, + request, + dispatch_headers(&original.headers), + streaming, + ) + .map(|(_, request)| request) + .map_err(|error| FlowError::Internal(error.to_string())) + } + + fn emit_requested(&self, request: &LlmRequest, inbound: WireProtocol, attempt: u32) { emit_mark( "switchyard.routing.requested", json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "routing_attempt": request.attempt.routing_attempt, - "profile_id": request.decision_profile.profile_id, - "rollout_mode": self.config.mode.label(), + "algorithm": "random", + "routing_attempt": attempt, + "inbound_profile": inbound.label(), }), identity_metadata(request), ); } - fn emit_decision( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - observe_only: bool, - latency_ms: u64, - ) { + fn emit_decision(&self, decision: &dyn Decision, attempt: u32, metadata: Json) { emit_mark( "switchyard.routing.decision", json!({ - "decision_id": decision.decision_id, - "profile_id": request.decision_profile.profile_id, - "router": decision.router.name, - "router_version": decision.router.version, + "algorithm": "random", "routing_attempt": attempt, - "backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "selected_model": decision.route.target_model, - "target_protocol_profile": decision.route.target_protocol_profile, - "target_endpoint": decision.route.target_endpoint, - "confidence": decision.confidence, - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - "latency_ms": latency_ms, - "observe_only": observe_only, - "rollout_mode": self.config.mode.label(), + "semantic_target": decision.selected_model(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + "reasoning": decision.reasoning(), }), - identity_metadata(request), + metadata, ); } - fn emit_retry( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - reason: &str, - ) { + fn emit_retry(&self, attempt: u32, reason: &str, request: &LlmRequest) { emit_mark( "switchyard.routing.retry", - json!({"routing_attempt": attempt, "previous_route": decision.route.backend_id, "retry_reason": reason, "rollout_mode": self.config.mode.label()}), + json!({ + "algorithm": "random", + "routing_attempt": attempt, + "retry_reason": reason, + }), identity_metadata(request), ); } - fn emit_error(&self, request: Option<&RoutingRequest>, attempt: u32, class: &str, error: &str) { + fn emit_error(&self, attempt: u32, class: &str, error: &str, request: &LlmRequest) { emit_mark( "switchyard.routing.error", - json!({"routing_attempt": attempt, "error_class": class, "error": error, "rollout_mode": self.config.mode.label()}), - request.map(identity_metadata).unwrap_or_else(|| json!({})), + json!({ + "algorithm": "random", + "routing_attempt": attempt, + "error_class": class, + "error": error, + }), + identity_metadata(request), ); } @@ -1339,190 +884,118 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.fallback", json!({ + "algorithm": "random", "fallback_reason": reason, "fallback_route": self.config.default_targets.target(inbound), "inbound_profile": inbound.label(), - "rollout_mode": self.config.mode.label(), }), - identity_metadata_from_request(request), + identity_metadata(request), ); } } -async fn check_switchyard_health( - client: &reqwest::Client, - health_url: &reqwest::Url, -) -> Result<(), String> { - let response = client - .get(health_url.clone()) - .send() - .await - .map_err(|error| { - format!("Switchyard service is required but health check {health_url} failed: {error}") - })?; - let status = response.status(); - if !status.is_success() { - return Err(format!( - "Switchyard service is required but health check {health_url} returned HTTP {status}" - )); - } - let body = response.json::().await.map_err(|error| { - format!("Switchyard health check {health_url} returned invalid JSON: {error}") - })?; - if body.get("status").and_then(Json::as_str) != Some("ok") { - return Err(format!( - "Switchyard health check {health_url} did not report status=ok" - )); - } - Ok(()) +struct RunFailure { + error: LibsyError, + provider_error: Option, } -fn switchyard_health_url(decision_api_url: &str) -> Result { - let mut url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - url.set_path(SWITCHYARD_HEALTH_PATH); - url.set_query(None); - url.set_fragment(None); - Ok(url) +impl RunFailure { + fn new(error: LibsyError, provider_error: &Arc>>) -> Self { + Self { + error, + provider_error: provider_error.lock().ok().and_then(|error| error.clone()), + } + } + + fn is_retryable(&self) -> bool { + self.provider_error + .as_ref() + .is_some_and(flow_error_is_retryable) + || libsy_error_is_retryable(&self.error) + } } -fn validate_atof_endpoint_name(name: Option<&str>) -> Result, String> { - if let Some(name) = name { - if name.trim().is_empty() { - return Err("atof_endpoint_name must be non-empty when configured".into()); - } - if name != name.trim() { - return Err("atof_endpoint_name must not have leading or trailing whitespace".into()); +fn build_algorithm(config: &SwitchyardConfig) -> Result, String> { + let targets = config + .targets + .keys() + .map(|name| LlmTarget { + semantic_name: name.clone(), + llm_client: None, + }) + .collect::>(); + let weights = config + .targets + .values() + .map(|target| target.weight) + .collect::>(); + match &config.algorithm { + AlgorithmConfig::Random { seed } => { + Random::new(LlmTargetSet::new(targets), Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) } } - Ok(name) } fn validate_config(config: &SwitchyardConfig) -> Result<(), String> { - validate_scalar_config(config)?; - validate_decision_api_url(&config.decision_api_url)?; - validate_target_bindings(config)?; - validate_default_targets(config) -} - -fn validate_scalar_config(config: &SwitchyardConfig) -> Result<(), String> { - if config.version != 1 { + if config.version != 2 { return Err(format!( - "unsupported Switchyard config version {}", + "unsupported Switchyard config version {}; version 1 used the removed switchyard-server Decision API, use version = 2", config.version )); } - if config.decision_profile_id.trim().is_empty() { - return Err("decision_profile_id must be non-empty".into()); - } - if config.decision_timeout_millis == 0 { - return Err("decision_timeout_millis must be greater than zero".into()); - } if config.max_retries > 10 { return Err("max_retries must not exceed 10".into()); } - if config.recent_message_count == 0 { - return Err("recent_message_count must be greater than zero".into()); - } - let atof_endpoint_name = validate_atof_endpoint_name(config.atof_endpoint_name.as_deref())?; - if config.context_mode == ContextMode::AtofRequired && atof_endpoint_name.is_none() { - return Err("atof_required Switchyard profiles require atof_endpoint_name".into()); - } - Ok(()) -} - -fn validate_decision_api_url(decision_api_url: &str) -> Result<(), String> { - let url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - if !matches!(url.scheme(), "http" | "https") { - return Err("decision_api_url must use http or https".into()); - } - Ok(()) -} - -fn validate_target_bindings(config: &SwitchyardConfig) -> Result<(), String> { if config.targets.is_empty() { return Err("targets must not be empty".into()); } if config.enabled_inbound_profiles.is_empty() { return Err("enabled_inbound_profiles must not be empty".into()); } - let mut exact_bindings = BTreeSet::new(); - for (id, target) in &config.targets { - if id.trim().is_empty() - || target.model.trim().is_empty() - || target.endpoint.trim().is_empty() - { - return Err("target IDs, models, and endpoints must be non-empty".into()); - } - let base_url = reqwest::Url::parse(&target.base_url) - .map_err(|error| format!("target {id:?} base_url is invalid: {error}"))?; - if !matches!(base_url.scheme(), "http" | "https") { - return Err(format!("target {id:?} base_url must use http or https")); + for (name, target) in &config.targets { + if name.trim().is_empty() || target.model.trim().is_empty() { + return Err("target names and models must be non-empty".into()); } if target.endpoint != target.protocol.endpoint() { return Err(format!( - "target {id:?} endpoint must be {:?} for {}", + "target {name:?} endpoint must be {:?} for {}", target.protocol.endpoint(), target.protocol.label() )); } - if !exact_bindings.insert(( - target.model.clone(), - target.protocol, - target.endpoint.clone(), - target.base_url.trim_end_matches('/').to_string(), - )) { + validate_http_url(&target.base_url, name)?; + if !target.weight.is_finite() || target.weight < 0.0 { return Err(format!( - "target {id:?} conflicts with another exact backend binding" + "target {name:?} weight must be finite and nonnegative" )); } } - Ok(()) -} - -fn validate_default_targets(config: &SwitchyardConfig) -> Result<(), String> { for &protocol in &config.enabled_inbound_profiles { - let id = config.default_targets.target(protocol); + let fallback = config.default_targets.target(protocol); let target = config .targets - .get(id) - .ok_or_else(|| format!("default target {id:?} is not configured"))?; + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; if target.protocol != protocol { return Err(format!( - "default target {id:?} must use protocol {}", + "default target {fallback:?} must use protocol {}", protocol.label() )); } } - Ok(()) + build_algorithm(config).map(|_| ()) } -fn resolve_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result { - let mut headers = HeaderMap::new(); - for (name, value) in static_headers { - insert_http_header(&mut headers, name, value)?; +fn validate_http_url(url: &str, target: &str) -> Result<(), String> { + let uri = url + .parse::() + .map_err(|error| format!("target {target:?} base_url is invalid: {error}"))?; + if !matches!(uri.scheme_str(), Some("http" | "https")) || uri.authority().is_none() { + return Err(format!("target {target:?} base_url must use http or https")); } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - insert_http_header(&mut headers, name, &value)?; - } - Ok(headers) + Ok(()) } fn resolve_json_headers( @@ -1531,6 +1004,7 @@ fn resolve_json_headers( ) -> Result, String> { let mut headers = Map::new(); for (name, value) in static_headers { + validate_target_header(name, value)?; headers.insert(name.clone(), Json::String(value.clone())); } for (name, variable) in environment_headers { @@ -1547,41 +1021,103 @@ fn resolve_json_headers( if value.trim().is_empty() { return Err(format!("environment variable {variable:?} is blank")); } + validate_target_header(name, &value)?; headers.insert(name.clone(), Json::String(value)); } Ok(headers) } -fn insert_http_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), String> { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid header name: {error}"))?; - let value = - HeaderValue::from_str(value).map_err(|error| format!("invalid header value: {error}"))?; - headers.insert(name, value); +fn validate_target_header(name: &str, value: &str) -> Result<(), String> { + let normalized = name.to_ascii_lowercase(); + if matches!( + normalized.as_str(), + INTERNAL_DISPATCH_URL_HEADER | INTERNAL_DISPATCH_ROUTE_HEADER | INTERNAL_RETRY_AWARE_HEADER + ) { + return Err(format!( + "target header {name:?} is reserved for Relay dispatch" + )); + } + HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; Ok(()) } -fn protocol_from_label(label: &str) -> FlowResult { - match label { - "openai_chat" | "openai_chat_completions" | "openai_chat_completions.v1" => { - Ok(WireProtocol::OpenaiChat) +fn response_metadata(source: Option<&Metadata>, protocol: WireProtocol) -> Metadata { + let mut metadata = source.cloned().unwrap_or_default(); + metadata.wire_format = Some(wire_format(protocol)); + metadata +} + +fn context_values(metadata: Option<&Metadata>, attempt: u32) -> BTreeMap { + let mut values = BTreeMap::from([ + ("relay.routing_attempt".into(), attempt.to_string()), + ("relay.scope_id".into(), task_scope_top().uuid.to_string()), + ]); + if let Some(metadata) = metadata { + for (key, value) in [ + ("relay.session_id", metadata.session_id.as_deref()), + ("relay.agent_id", metadata.agent_id.as_deref()), + ("relay.turn_id", metadata.turn_id.as_deref()), + ("relay.correlation_id", metadata.correlation_id.as_deref()), + ] { + if let Some(value) = value { + values.insert(key.into(), value.into()); + } } - "openai_responses" | "openai_responses.v1" => Ok(WireProtocol::OpenaiResponses), - "anthropic_messages" | "anthropic_messages.v1" => Ok(WireProtocol::AnthropicMessages), - value => Err(FlowError::InvalidArgument(format!( - "unsupported Switchyard target protocol {value:?}" - ))), } + values +} + +fn string_headers(headers: &Map) -> BTreeMap { + headers + .iter() + .filter_map(|(name, value)| value.as_str().map(|value| (name.clone(), value.into()))) + .collect() +} + +fn dispatch_headers(headers: &Map) -> Map { + headers + .iter() + .filter(|(name, _)| { + let normalized = name.to_ascii_lowercase(); + !matches!( + normalized.as_str(), + "authorization" | "proxy-authorization" | "x-api-key" | "api-key" + ) && !normalized.starts_with("x-nemo-relay-") + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +fn relay_metadata(headers: &BTreeMap) -> Metadata { + let mut metadata = Metadata::from_headers(headers); + if metadata.turn_id.is_none() { + metadata.turn_id = header_value(headers, RELAY_TURN_ID_HEADER).map(ToOwned::to_owned); + } + if metadata.correlation_id.is_none() { + metadata.correlation_id = + header_value(headers, RELAY_REQUEST_ID_HEADER).map(ToOwned::to_owned); + } + metadata +} + +fn libsy_identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "request_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) } -fn header(request: &LlmRequest, name: &str) -> Option { - request - .headers - .get(name) - .and_then(Json::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) +fn header_value<'a>(headers: &'a BTreeMap, name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + .filter(|value| !value.trim().is_empty()) } fn dispatch_url(base_url: &str, endpoint: &str) -> String { @@ -1594,26 +1130,40 @@ fn dispatch_url(base_url: &str, endpoint: &str) -> String { format!("{base}{endpoint}") } -fn identity_metadata(request: &RoutingRequest) -> Json { - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "turn_id": request.identity.turn_id, - "owner_id": request.identity.owner_id, - }) +fn remember_provider_error(slot: &Arc>>, error: &FlowError) { + if let Ok(mut stored) = slot.lock() { + *stored = Some(error.clone()); + } } -fn identity_metadata_from_request(request: &LlmRequest) -> Json { - json!({ - "session_id": header(request, "x-nemo-relay-session-id"), - "request_id": header(request, "x-nemo-relay-request-id"), - "turn_id": header(request, "x-nemo-relay-turn-id"), - "owner_id": header(request, "x-nemo-relay-owner-id"), - }) +fn flow_error_is_retryable(error: &FlowError) -> bool { + matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) } -fn error_is_retryable(error: &FlowError) -> bool { - matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) +fn libsy_error_is_retryable(error: &LibsyError) -> bool { + match error { + LibsyError::ClientCall { source, .. } => match source { + LlmClientError::Transport { .. } + | LlmClientError::Timeout { .. } + | LlmClientError::ContextWindowExceeded { .. } => true, + LlmClientError::UpstreamHttp { status, .. } => { + matches!(*status, 408 | 409 | 425 | 429) || *status >= 500 + } + _ => false, + }, + _ => false, + } +} + +fn identity_metadata(request: &LlmRequest) -> Json { + let headers = string_headers(&request.headers); + let metadata = relay_metadata(&headers); + json!({ + "session_id": metadata.session_id, + "agent_id": metadata.agent_id, + "turn_id": metadata.turn_id, + "request_id": metadata.correlation_id, + }) } fn emit_mark(name: &str, data: Json, metadata: Json) { @@ -1624,7 +1174,7 @@ fn emit_mark(name: &str, data: Json, metadata: Json) { .data_schema( DataSchema::builder() .name(ROUTING_MARK_SCHEMA) - .version("1") + .version("2") .build(), ) .metadata(metadata) @@ -1636,184 +1186,6 @@ fn emit_mark(name: &str, data: Json, metadata: Json) { } } -fn emit_terminal_error(error: &FlowError, phase: &str, rollout_mode: &str, metadata: Json) { - emit_mark( - "switchyard.routing.terminal_error", - json!({"error_class": provider_error_class(error), "error": provider_error_summary(error), "phase": phase, "rollout_mode": rollout_mode}), - metadata, - ); -} - -fn provider_error_class(error: &FlowError) -> &'static str { - match error { - FlowError::Upstream(failure) => match failure.class { - nemo_relay::error::UpstreamFailureClass::Connection => "connection", - nemo_relay::error::UpstreamFailureClass::Timeout => "timeout", - nemo_relay::error::UpstreamFailureClass::RetryableStatus => "retryable_status", - nemo_relay::error::UpstreamFailureClass::ContextWindow => "context_window", - nemo_relay::error::UpstreamFailureClass::ModelUnavailable => "model_unavailable", - nemo_relay::error::UpstreamFailureClass::Authentication => "authentication", - nemo_relay::error::UpstreamFailureClass::InvalidRequest => "invalid_request", - nemo_relay::error::UpstreamFailureClass::Other => "other", - }, - _ => "relay", - } -} - -fn provider_error_summary(error: &FlowError) -> String { - match error { - FlowError::Upstream(failure) => match failure.status { - Some(status) => format!("{}:http_{status}", provider_error_class(error)), - None => provider_error_class(error).to_string(), - }, - _ => error.to_string(), - } -} - -struct PrefixedStream { - first: Option>, - upstream: LlmJsonStream, -} - -impl futures_util::Stream for PrefixedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(first) = self.first.take() { - Poll::Ready(Some(first)) - } else { - Pin::new(&mut self.upstream).poll_next(cx) - } - } -} - -impl LlmStreamInner for PrefixedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.first = None; - Box::pin(async move { this.upstream.close().await }) - } -} - -struct TerminalMarkedStream { - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, - finished: bool, -} - -impl futures_util::Stream for TerminalMarkedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.finished { - return Poll::Ready(None); - } - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => Poll::Ready(Some(Ok(chunk))), - Poll::Ready(Some(Err(error))) => { - self.finished = true; - emit_terminal_error(&error, self.phase, self.rollout_mode, self.metadata.clone()); - Poll::Ready(Some(Err(error))) - } - Poll::Ready(None) => { - self.finished = true; - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } -} - -impl LlmStreamInner for TerminalMarkedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - Box::pin(async move { self.get_mut().upstream.close().await }) - } -} - -struct TranslatedStream { - upstream: LlmJsonStream, - transcoder: StreamTranscoder, - buffered: VecDeque>, - upstream_finished: bool, -} - -impl futures_util::Stream for TranslatedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - loop { - if let Some(chunk) = self.buffered.pop_front() { - return Poll::Ready(Some(chunk)); - } - if self.upstream_finished { - return Poll::Ready(None); - } - - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => match self.transcoder.transcode(&chunk) { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - }, - Poll::Ready(Some(Err(error))) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - Poll::Ready(None) => { - self.upstream_finished = true; - match self.transcoder.finish() { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => return Poll::Ready(Some(Err(error))), - } - } - Poll::Pending => return Poll::Pending, - } - } - } -} - -impl LlmStreamInner for TranslatedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.buffered.clear(); - this.upstream_finished = true; - Box::pin(async move { this.upstream.close().await }) - } -} - -fn mark_terminal_stream( - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TerminalMarkedStream { - upstream, - phase, - rollout_mode, - metadata, - finished: false, - }) -} - -fn translated_stream( - source: WireProtocol, - target: WireProtocol, - effective_model: String, - upstream: LlmJsonStream, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TranslatedStream { - upstream, - transcoder: StreamTranscoder::new(source, target, effective_model), - buffered: VecDeque::new(), - upstream_finished: false, - }) -} - #[cfg(test)] #[path = "../tests/unit/component_tests.rs"] mod tests; diff --git a/crates/switchyard/src/contract.rs b/crates/switchyard/src/contract.rs deleted file mode 100644 index 675234ca1..000000000 --- a/crates/switchyard/src/contract.rs +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! JSON compatibility types for the Switchyard Decision API. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// Routing request schema supported by this plugin. -pub const ROUTING_REQUEST_SCHEMA_VERSION: &str = "switchyard.routing_request.v1"; -/// Routing decision schema supported by this plugin. -pub const ROUTING_DECISION_SCHEMA_VERSION: &str = "switchyard.routing_decision.v1"; - -/// Request-time materialization supplied to Switchyard. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RequestMaterialization { - /// Identity, protocol, summary, and attempt only. - None, - /// Baseline summary without current request material. - SummaryOnly, - /// Latest user prompt. - LatestUserPrompt, - /// Bounded recent message window. - RecentMessageWindow, - /// Relay-normalized request plus its provider body. - AnnotatedRequest, - /// Complete provider request body. - FullBody, -} - -/// Switchyard profile selection. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProfile { - /// Profile ID loaded by Switchyard. - pub profile_id: String, - /// Request materialization mode. - pub request_materialization: RequestMaterialization, -} - -/// Normalized Relay identity. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestIdentity { - /// Stable session identifier. - pub session_id: String, - /// Per-request identifier. - pub request_id: String, - /// Optional turn identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub turn_id: Option, - /// Optional parent scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_scope_id: Option, - /// Optional root scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub root_scope_id: Option, - /// Harness name. - pub harness: String, - /// Request source. - pub source: String, - /// Optional resolved work owner. - #[serde(skip_serializing_if = "Option::is_none")] - pub owner_id: Option, - /// Native, explicit, or synthetic identity quality. - pub quality: String, -} - -/// Inbound protocol context. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestProtocol { - /// Inbound protocol profile. - pub inbound_profile: String, - /// Inbound endpoint. - pub inbound_endpoint: String, - /// Response profile expected by the harness. - pub desired_response_profile: String, -} - -/// Cheap provider-request summary. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct RequestSummary { - /// Client-requested model. - #[serde(skip_serializing_if = "Option::is_none")] - pub client_requested_model: Option, - /// Optional prompt-token estimate. - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_token_estimate: Option, - /// Number of tools in the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_count_in_payload: Option, - /// Whether a system prompt is present. - #[serde(skip_serializing_if = "Option::is_none")] - pub has_system_prompt: Option, -} - -/// Routing-attempt context. Additive fields are ignored by older Switchyard servers. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionAttempt { - /// One-indexed routing attempt. - pub routing_attempt: u32, - /// Maximum Decision API attempts. - pub max_routing_attempts: u32, - /// Previously selected backend. - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_route: Option, - /// Reason another decision is requested. - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_reason: Option, -} - -/// Canonical routing request sent by Relay. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingRequest { - /// Schema identifier. - pub schema_version: String, - /// Switchyard profile selection. - pub decision_profile: DecisionProfile, - /// Normalized identity. - pub identity: RequestIdentity, - /// Inbound protocol. - pub protocol: RequestProtocol, - /// Cheap request summary. - pub request_summary: RequestSummary, - /// Optional request materialization. - #[serde(skip_serializing_if = "Option::is_none")] - pub current_request: Option, - /// Attempt metadata. - pub attempt: DecisionAttempt, -} - -/// Router metadata returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProvider { - /// Router name. - pub name: String, - /// Router version. - pub version: String, -} - -/// Selected target returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingTarget { - /// Tier label. - pub tier: String, - /// Selected model. - pub target_model: String, - /// Backend binding ID. - pub backend_id: String, - /// Target protocol profile. - pub target_protocol_profile: String, - /// Target endpoint. - pub target_endpoint: String, -} - -/// Canonical Switchyard routing decision. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingDecision { - /// Schema identifier. - pub schema_version: String, - /// Decision identifier. - pub decision_id: String, - /// Router metadata. - pub router: DecisionProvider, - /// Selected target. - pub route: RoutingTarget, - /// Explicit counterfactual route supplied by routers that define one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub baseline_route: Option, - /// Optional confidence. - #[serde(default)] - pub confidence: Option, - /// Optional reason code. - #[serde(default)] - pub reason_code: Option, - /// Optional reason summary. - #[serde(default)] - pub reason_summary: Option, - /// Additive router metadata. - #[serde(default)] - pub metadata: BTreeMap, - /// Unknown additive response fields. - #[serde(flatten)] - pub extra: BTreeMap, -} diff --git a/crates/switchyard/src/lib.rs b/crates/switchyard/src/lib.rs index 238d4337f..a0168e4e9 100644 --- a/crates/switchyard/src/lib.rs +++ b/crates/switchyard/src/lib.rs @@ -3,15 +3,13 @@ #![deny(rustdoc::broken_intra_doc_links, rustdoc::private_intra_doc_links)] -//! First-party Switchyard Decision API routing plugin for NeMo Relay. +//! First-party in-process Switchyard libsy routing plugin for NeMo Relay. pub mod component; -pub mod contract; mod stream_translation; mod translation; pub use component::{ - ContextMode, ProtocolDefaults, RoutingMode, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, - TargetBinding, WireProtocol, deregister_switchyard_component, register_switchyard_component, - validate_switchyard_atof_configuration, + AlgorithmConfig, ProtocolDefaults, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, TargetBinding, + WireProtocol, deregister_switchyard_component, register_switchyard_component, }; diff --git a/crates/switchyard/src/stream_translation.rs b/crates/switchyard/src/stream_translation.rs index 808834a2b..6e5e846a9 100644 --- a/crates/switchyard/src/stream_translation.rs +++ b/crates/switchyard/src/stream_translation.rs @@ -1,94 +1,386 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Relay stream adapter for Switchyard's provider translation engine. +//! Relay stream adapters for Switchyard provider events. -use nemo_relay::error::{FlowError, Result}; +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; + +use futures_util::Stream; +use nemo_relay::api::runtime::{LlmJsonStream, LlmStreamInner}; +use nemo_relay::error::{FlowError, Result as FlowResult, UpstreamFailure, UpstreamFailureClass}; use serde_json::Value as Json; +use switchyard_protocol::{LlmClientError, LlmResponseChunk, LlmResponseStream, WireFormat}; use switchyard_translation::{StreamTranslationState, TranslationEngine}; +use tokio::sync::oneshot; + +/// Completion handles for provider streams owned by one libsy run. +#[derive(Clone, Default)] +pub(crate) struct StreamCloseTracker { + receivers: Arc>>>>, +} + +impl StreamCloseTracker { + fn track(&self, receiver: oneshot::Receiver>) { + if let Ok(mut receivers) = self.receivers.lock() { + receivers.push(receiver); + } + } -use crate::component::WireProtocol; -use crate::translation::wire_format; + async fn close_all(&self) -> FlowResult<()> { + let receivers = self + .receivers + .lock() + .map_err(|error| FlowError::Internal(error.to_string()))? + .drain(..) + .collect::>(); + for receiver in receivers { + match receiver.await { + Ok(result) => result?, + Err(_) => { + return Err(FlowError::Internal( + "Switchyard provider stream close task was dropped".into(), + )); + } + } + } + Ok(()) + } +} -pub(crate) struct StreamTranscoder { - engine: TranslationEngine, - source: WireProtocol, - target: WireProtocol, +/// Converts one Relay provider stream into libsy's neutral stream contract. +pub(crate) fn provider_response_stream( + upstream: LlmJsonStream, + source: WireFormat, + first: Json, + tracker: StreamCloseTracker, + model: String, +) -> std::result::Result { + let translation = TranslationEngine::default(); + let mut state = StreamTranslationState::new(source, source); + let first = decode_provider_event(&translation, &mut state, source, first)?; + let (closed_tx, closed_rx) = oneshot::channel(); + tracker.track(closed_rx); + Ok(Box::pin(RelayProviderStream { + upstream: Some(upstream), + first: Some(first), + source, + translation, + state, + closed_tx: Some(closed_tx), + model, + })) +} + +struct RelayProviderStream { + upstream: Option, + first: Option, + source: WireFormat, + translation: TranslationEngine, state: StreamTranslationState, + closed_tx: Option>>, + model: String, } -impl StreamTranscoder { - pub(crate) fn new( - source: WireProtocol, - target: WireProtocol, - effective_model: impl Into, - ) -> Self { - let effective_model = effective_model.into(); - let mut state = StreamTranslationState::new(wire_format(source), wire_format(target)); - state.model = Some(effective_model.clone()); - state.target_model = Some(effective_model); - Self { - engine: TranslationEngine::default(), - source, - target, - state, +impl RelayProviderStream { + fn decode(&mut self, raw: Json) -> std::result::Result { + decode_provider_event(&self.translation, &mut self.state, self.source, raw) + } +} + +fn decode_provider_event( + translation: &TranslationEngine, + state: &mut StreamTranslationState, + source: WireFormat, + raw: Json, +) -> std::result::Result { + let chunk = translation + .decode_stream_event(state, source, &raw) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + ensure_provider_event_succeeded(&chunk)?; + Ok(chunk) +} + +fn ensure_provider_event_succeeded( + chunk: &LlmResponseChunk, +) -> std::result::Result<(), LlmClientError> { + match chunk { + LlmResponseChunk::ProviderEvent { normalized, .. } => { + for chunk in normalized { + ensure_provider_event_succeeded(chunk)?; + } + Ok(()) + } + LlmResponseChunk::DecodeError { message } => { + Err(LlmClientError::ResponseTranslation(message.clone())) } + LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp { + status: 502, + body: message.clone(), + }), + _ => Ok(()), } +} - pub(crate) fn transcode(&mut self, chunk: &Json) -> Result> { - if unsupported_stream_chunk(self.source, chunk) { - return Err(FlowError::InvalidArgument( - "provider-specific streaming extension cannot be translated safely".into(), - )); +impl Stream for RelayProviderStream { + type Item = std::result::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + if let Some(first) = this.first.take() { + return Poll::Ready(Some(Ok(first))); + } + let Some(upstream) = this.upstream.as_mut() else { + return Poll::Ready(None); + }; + match Pin::new(upstream).poll_next(cx) { + Poll::Ready(Some(Ok(raw))) => Poll::Ready(Some(this.decode(raw))), + Poll::Ready(Some(Err(error))) => { + Poll::Ready(Some(Err(flow_to_client_error(error, &this.model)))) + } + Poll::Ready(None) => { + this.upstream.take(); + if let Some(closed_tx) = this.closed_tx.take() { + let _ = closed_tx.send(Ok(())); + } + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, } - self.engine - .translate_event( - &mut self.state, - wire_format(self.source), - wire_format(self.target), - chunk, - ) - .map_err(|error| { - FlowError::InvalidArgument(format!("Switchyard stream translation failed: {error}")) - }) } +} - pub(crate) fn finish(&mut self) -> Result> { - self.engine - .finish_stream(&mut self.state, wire_format(self.target)) - .map_err(|error| { - FlowError::InvalidArgument(format!( - "Switchyard stream finalization failed: {error}" - )) - }) +impl Drop for RelayProviderStream { + fn drop(&mut self) { + let Some(mut upstream) = self.upstream.take() else { + return; + }; + let closed_tx = self.closed_tx.take(); + tokio::spawn(async move { + let result = upstream.close().await; + if let Some(closed_tx) = closed_tx { + let _ = closed_tx.send(result); + } + }); } } -fn unsupported_stream_chunk(source: WireProtocol, chunk: &Json) -> bool { - match source { - WireProtocol::OpenaiChat => { - chunk["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["delta"].get("audio").is_some() - || choice["delta"].get("reasoning_content").is_some() - }) +/// Converts libsy's final response stream back into Relay provider events. +pub(crate) fn relay_response_stream( + upstream: LlmResponseStream, + source: WireFormat, + target: WireFormat, + tracker: StreamCloseTracker, +) -> LlmJsonStream { + LlmJsonStream::from_closeable(LibsyResponseStream { + upstream: Some(upstream), + source, + target, + translation: TranslationEngine::default(), + state: StreamTranslationState::new(source, target), + buffered: VecDeque::new(), + finished: false, + tracker, + }) +} + +struct LibsyResponseStream { + upstream: Option, + source: WireFormat, + target: WireFormat, + translation: TranslationEngine, + state: StreamTranslationState, + buffered: VecDeque>, + finished: bool, + tracker: StreamCloseTracker, +} + +impl Stream for LibsyResponseStream { + type Item = FlowResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + loop { + if let Some(item) = this.buffered.pop_front() { + return Poll::Ready(Some(item)); + } + if this.finished { + return Poll::Ready(None); + } + let Some(upstream) = this.upstream.as_mut() else { + this.finished = true; + return Poll::Ready(None); + }; + match upstream.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(event))) => { + match this + .translation + .encode_stream_event(&mut this.state, this.target, event) + { + Ok(events) => { + this.buffered.extend(events.into_iter().map(Ok)); + } + Err(error) => { + this.finished = true; + return Poll::Ready(Some(Err(FlowError::InvalidArgument(format!( + "Switchyard stream translation failed: {error}" + ))))); + } + } + } + Poll::Ready(Some(Err(error))) => { + this.finished = true; + return Poll::Ready(Some(Err(client_to_flow_error(error)))); + } + Poll::Ready(None) => { + this.upstream.take(); + this.finished = true; + if this.source == this.target { + return Poll::Ready(None); + } + match this.translation.finish_stream(&mut this.state, this.target) { + Ok(events) => { + this.buffered.extend(events.into_iter().map(Ok)); + } + Err(error) => { + return Poll::Ready(Some(Err(FlowError::InvalidArgument(format!( + "Switchyard stream finalization failed: {error}" + ))))); + } + } + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +impl LlmStreamInner for LibsyResponseStream { + fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { + let this = self.get_mut(); + this.upstream.take(); + this.buffered.clear(); + this.finished = true; + Box::pin(async move { this.tracker.close_all().await }) + } +} + +pub(crate) fn flow_to_client_error(error: FlowError, model: &str) -> LlmClientError { + match error { + FlowError::Upstream(failure) => match failure.class { + UpstreamFailureClass::Timeout => LlmClientError::Timeout { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + failure.body, + )), + }, + UpstreamFailureClass::Connection => LlmClientError::Transport { + source: Box::new(std::io::Error::other(failure.body)), + }, + UpstreamFailureClass::ContextWindow => LlmClientError::ContextWindowExceeded { + model: model.into(), + message: failure.body, + }, + UpstreamFailureClass::ModelUnavailable => LlmClientError::General(failure.body), + UpstreamFailureClass::InvalidRequest => LlmClientError::InvalidRequest { + message: failure.body, + }, + UpstreamFailureClass::Authentication | UpstreamFailureClass::Other + if failure.status.is_none() => + { + LlmClientError::General(failure.body) + } + _ => LlmClientError::UpstreamHttp { + status: failure.status.unwrap_or(500), + body: failure.body, + }, + }, + FlowError::InvalidArgument(message) => LlmClientError::InvalidRequest { message }, + other => LlmClientError::General(other.to_string()), + } +} + +fn client_to_flow_error(error: LlmClientError) -> FlowError { + match error { + LlmClientError::Transport { source } => { + provider_failure(None, source.to_string(), UpstreamFailureClass::Connection) } - WireProtocol::OpenaiResponses => { - chunk.get("type").and_then(Json::as_str) == Some("response.output_item.added") - && matches!( - chunk["item"].get("type").and_then(Json::as_str), - Some("reasoning" | "computer_call" | "web_search_call") - ) + LlmClientError::Timeout { source } => { + provider_failure(None, source.to_string(), UpstreamFailureClass::Timeout) } - WireProtocol::AnthropicMessages => { - chunk.get("type").and_then(Json::as_str) == Some("content_block_start") - && !matches!( - chunk["content_block"].get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) + LlmClientError::ContextWindowExceeded { message, .. } => { + provider_failure(None, message, UpstreamFailureClass::ContextWindow) } + LlmClientError::UpstreamHttp { status, body } => { + let class = if status == 401 || status == 403 { + UpstreamFailureClass::Authentication + } else if matches!(status, 408 | 409 | 425 | 429) || status >= 500 { + UpstreamFailureClass::RetryableStatus + } else if (400..500).contains(&status) { + UpstreamFailureClass::InvalidRequest + } else { + UpstreamFailureClass::Other + }; + provider_failure(Some(status), body, class) + } + LlmClientError::InvalidRequest { message } + | LlmClientError::RequestTranslation(message) + | LlmClientError::RequestEncoding(message) + | LlmClientError::ResponseTranslation(message) + | LlmClientError::Configuration { message } => FlowError::InvalidArgument(message), + other => FlowError::Internal(format!("Switchyard model call failed: {other}")), + } +} + +fn provider_failure(status: Option, body: String, class: UpstreamFailureClass) -> FlowError { + FlowError::Upstream(UpstreamFailure { + status, + body, + headers: Default::default(), + class, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::StreamExt; + use serde_json::json; + + #[tokio::test] + async fn preserved_translation_replays_unknown_same_protocol_stream_fields() { + let raw = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "provider/model", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "hello"}, + "finish_reason": null + }] + }); + let upstream = LlmJsonStream::new(futures_util::stream::iter([Ok(raw.clone())])); + let tracker = StreamCloseTracker::default(); + let neutral = provider_response_stream( + upstream, + WireFormat::OpenAiChat, + raw.clone(), + tracker.clone(), + "target".into(), + ) + .unwrap(); + let mut replay = relay_response_stream( + neutral, + WireFormat::OpenAiChat, + WireFormat::OpenAiChat, + tracker, + ); + let output = replay.next().await.and_then(Result::ok); + + assert_eq!(output, Some(raw)); } } diff --git a/crates/switchyard/src/translation.rs b/crates/switchyard/src/translation.rs index d1640226c..689e9890f 100644 --- a/crates/switchyard/src/translation.rs +++ b/crates/switchyard/src/translation.rs @@ -1,13 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +//! Relay adapters for Switchyard's buffered translation engine. + use nemo_relay::api::llm::LlmRequest; use nemo_relay::error::{FlowError, Result}; use serde_json::{Map, Value as Json}; +use switchyard_protocol::{AggLlmResponse, LlmRequest as ProtocolRequest}; use switchyard_translation::{ - ContentBlock, DeterministicIdPolicy, ImageSource, LlmRequest as TranslationRequest, - LossyConversionPolicy, PreservationPolicy, Role, TargetCapabilities, TranslationDiagnostic, - TranslationEngine, TranslationPolicy, UnknownFieldPolicy, WireFormat, + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, WireFormat, }; use crate::component::WireProtocol; @@ -20,7 +23,7 @@ pub(crate) fn decode_request( engine: &TranslationEngine, protocol: WireProtocol, request: &LlmRequest, -) -> Result { +) -> Result { let output = engine .decode_request( wire_format(protocol), @@ -28,106 +31,51 @@ pub(crate) fn decode_request( &translation_policy(), ) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(output.request) } pub(crate) fn encode_request( engine: &TranslationEngine, protocol: WireProtocol, - request: &TranslationRequest, + request: &ProtocolRequest, headers: Map, ) -> Result { let output = engine - .encode_request(wire_format(protocol), request, &translation_policy()) + .encode_request( + wire_format(protocol), + request, + &request_translation_policy(protocol), + ) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(LlmRequest { headers, content: output.body, }) } -pub(crate) fn validate_portable_request( +pub(crate) fn decode_response( engine: &TranslationEngine, protocol: WireProtocol, - request: &LlmRequest, -) -> Result<()> { - const RESTRICTED_KEYS: &[&str] = &[ - "cache_control", - "audio", - "thinking", - "computer_use", - "server_tool_use", - ]; - if protocol == WireProtocol::AnthropicMessages - && contains_invalid_anthropic_image_source(&request.content) - { - return Err(FlowError::InvalidArgument( - "request uses an unsupported or malformed Anthropic image source".into(), - )); - } - if contains_any_key_recursive(&request.content, RESTRICTED_KEYS) { - return Err(FlowError::InvalidArgument( - "request uses a provider-specific extension that requires same-protocol fail-open" - .into(), - )); - } - let decoded = decode_request(engine, protocol, request)?; - if decoded.reasoning.effort.is_some() - || decoded.reasoning.raw.is_some() - || decoded - .extensions - .fields - .iter() - .any(|(key, value)| key != "stream_options" || !portable_stream_options(value)) - || request_contains_unsupported_content(&decoded) - { - return Err(FlowError::InvalidArgument( - "request uses provider-specific fields that cannot be translated safely".into(), - )); - } - Ok(()) -} - -pub(crate) fn latest_user_prompt(request: &TranslationRequest) -> Option { - request - .messages - .iter() - .rev() - .find(|message| message.role == Role::User) - .and_then(|message| message.text_content("\n")) -} - -pub(crate) fn recent_message_window( - request: &TranslationRequest, - count: usize, -) -> TranslationRequest { - let mut window = request.clone(); - let split = window.messages.len().saturating_sub(count); - window.messages = window.messages.split_off(split); - window + response: &Json, +) -> Result { + let output = engine + .decode_response(wire_format(protocol), response, &translation_policy()) + .map_err(translation_error)?; + ensure_safe_diagnostics(&output.diagnostics)?; + Ok(output.response) } -pub(crate) fn translate_response( +pub(crate) fn encode_response( engine: &TranslationEngine, - source: WireProtocol, - target: WireProtocol, - response: &Json, + protocol: WireProtocol, + response: &AggLlmResponse, ) -> Result { - if source == target { - return Ok(response.clone()); - } - ensure_portable_response(source, response)?; let output = engine - .translate_response( - wire_format(source), - wire_format(target), - response, - &translation_policy(), - ) + .encode_response(wire_format(protocol), response, &translation_policy()) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(output.body) } @@ -141,176 +89,241 @@ pub(crate) const fn wire_format(protocol: WireProtocol) -> WireFormat { fn translation_policy() -> TranslationPolicy { TranslationPolicy { - unknown_field_policy: UnknownFieldPolicy::Reject, + unknown_field_policy: UnknownFieldPolicy::Preserve, lossy_conversion_policy: LossyConversionPolicy::Reject, deterministic_ids: DeterministicIdPolicy::GenerateStable { prefix: "relay".into(), }, - preservation: PreservationPolicy::Disabled, + preservation: PreservationPolicy::InMemory, target_capabilities: TargetCapabilities::default(), } } +fn request_translation_policy(protocol: WireProtocol) -> TranslationPolicy { + let mut policy = translation_policy(); + if protocol == WireProtocol::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + fn translation_error(error: switchyard_translation::TranslationError) -> FlowError { FlowError::InvalidArgument(format!("Switchyard translation failed: {error}")) } -fn ensure_no_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { - if diagnostics.is_empty() { +fn ensure_safe_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { + let lossy = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if lossy.is_empty() { Ok(()) } else { Err(FlowError::InvalidArgument(format!( - "Switchyard translation was not lossless: {diagnostics:?}" + "Switchyard translation was not lossless: {lossy:?}" ))) } } -fn portable_stream_options(value: &Json) -> bool { - let Some(options) = value.as_object() else { - return false; - }; - options.len() == 1 && options.get("include_usage").is_some_and(Json::is_boolean) -} +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; -fn request_contains_unsupported_content(request: &TranslationRequest) -> bool { - request - .instructions - .iter() - .flat_map(|instruction| instruction.content.iter()) - .chain( - request - .messages - .iter() - .flat_map(|message| message.content.iter()), - ) - .any(unsupported_content_block) -} + const PROTOCOLS: [WireProtocol; 3] = [ + WireProtocol::OpenaiChat, + WireProtocol::OpenaiResponses, + WireProtocol::AnthropicMessages, + ]; -fn unsupported_content_block(block: &ContentBlock) -> bool { - match block { - ContentBlock::Text { .. } | ContentBlock::Refusal { .. } | ContentBlock::ToolCall(_) => { - false - } - ContentBlock::Image { source } => invalid_image_source(source), - ContentBlock::ToolResult(result) => result.content.iter().any(unsupported_content_block), - ContentBlock::Reasoning { .. } - | ContentBlock::Audio { .. } - | ContentBlock::Video { .. } - | ContentBlock::File { .. } - | ContentBlock::Unknown { .. } => true, + #[test] + fn same_protocol_buffered_response_preserves_provider_extensions() { + let engine = translation_engine(); + let original = json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": "provider/model", + "system_fingerprint": "fp_exact", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop" + }] + }); + + let decoded = decode_response(&engine, WireProtocol::OpenaiChat, &original); + assert!(decoded.is_ok()); + let encoded = decoded + .and_then(|response| encode_response(&engine, WireProtocol::OpenaiChat, &response)); + assert_eq!(encoded.ok(), Some(original)); } -} -fn invalid_image_source(source: &ImageSource) -> bool { - match source { - ImageSource::Url { url, .. } => { - url.starts_with("data:") && base64_data_uri_parts(url).is_none() - } - ImageSource::Base64 { media_type, data } => { - media_type.as_deref().is_none_or(str::is_empty) || data.is_empty() - } - ImageSource::Raw(_) => true, + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let engine = translation_engine(); + let original = LlmRequest { + headers: Map::new(), + content: json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + + let decoded = decode_request(&engine, WireProtocol::OpenaiChat, &original); + assert!(decoded.is_ok()); + let encoded = decoded.and_then(|request| { + encode_request(&engine, WireProtocol::OpenaiChat, &request, Map::new()) + }); + assert_eq!( + encoded.ok().map(|request| request.content), + Some(original.content) + ); } -} -fn base64_data_uri_parts(url: &str) -> Option<(&str, &str)> { - let (metadata, data) = url.strip_prefix("data:")?.split_once(',')?; - let media_type = metadata.strip_suffix(";base64")?; - (!media_type.is_empty() && !data.is_empty()).then_some((media_type, data)) -} + #[test] + fn all_same_protocol_requests_and_responses_preserve_unknown_fields() { + let engine = translation_engine(); + for protocol in PROTOCOLS { + let request = request_fixture(protocol, true); + let decoded = decode_request( + &engine, + protocol, + &LlmRequest { + headers: Map::new(), + content: request.clone(), + }, + ) + .expect("request should decode"); + let encoded = encode_request(&engine, protocol, &decoded, Map::new()) + .expect("request should encode"); + assert_eq!( + encoded.content, request, + "request mismatch for {protocol:?}" + ); -fn contains_invalid_anthropic_image_source(value: &Json) -> bool { - match value { - Json::Object(object) => { - if object.get("type").and_then(Json::as_str) == Some("image") { - let Some(source) = object.get("source").and_then(Json::as_object) else { - return true; - }; - match source.get("type").and_then(Json::as_str) { - Some("url") => source - .get("url") - .and_then(Json::as_str) - .is_none_or(str::is_empty), - Some("base64") => { - source - .get("media_type") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - || source - .get("data") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - } - _ => true, + let response = response_fixture(protocol, true); + let decoded = + decode_response(&engine, protocol, &response).expect("response should decode"); + let encoded = + encode_response(&engine, protocol, &decoded).expect("response should encode"); + assert_eq!(encoded, response, "response mismatch for {protocol:?}"); + } + } + + #[test] + fn every_cross_protocol_pair_translates_common_text_response_data() { + let engine = translation_engine(); + for source in PROTOCOLS { + for target in PROTOCOLS { + if source == target { + continue; } - } else { - object.values().any(contains_invalid_anthropic_image_source) + + let request = request_fixture(source, false); + let decoded = decode_request( + &engine, + source, + &LlmRequest { + headers: Map::new(), + content: request, + }, + ) + .expect("request should decode"); + let encoded = encode_request(&engine, target, &decoded, Map::new()) + .expect("request should encode"); + assert!( + encoded.content.to_string().contains("hello"), + "translated request lost text for {source:?} -> {target:?}: {}", + encoded.content + ); + + let response = response_fixture(source, false); + let decoded = + decode_response(&engine, source, &response).expect("response should decode"); + let encoded = + encode_response(&engine, target, &decoded).expect("response should encode"); + let encoded_text = encoded.to_string(); + assert!( + encoded_text.contains("world"), + "translated response lost text for {source:?} -> {target:?}: {encoded}" + ); + assert!( + encoded_text.contains('7') && encoded_text.contains('3'), + "translated response lost usage for {source:?} -> {target:?}: {encoded}" + ); } } - Json::Array(items) => items.iter().any(contains_invalid_anthropic_image_source), - _ => false, } -} -fn contains_any_key_recursive(value: &Json, keys: &[&str]) -> bool { - match value { - Json::Object(object) => { - object.keys().any(|key| keys.contains(&key.as_str())) - || object - .values() - .any(|value| contains_any_key_recursive(value, keys)) + fn request_fixture(protocol: WireProtocol, extension: bool) -> Json { + let mut body = match protocol { + WireProtocol::OpenaiChat => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), + WireProtocol::OpenaiResponses => json!({ + "model": "caller/model", + "input": [{ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}] + }], + "max_output_tokens": 32 + }), + WireProtocol::AnthropicMessages => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), + }; + if extension { + body["provider_extension"] = json!({"exact": true}); } - Json::Array(items) => items - .iter() - .any(|value| contains_any_key_recursive(value, keys)), - _ => false, + body } -} -fn ensure_portable_response(protocol: WireProtocol, response: &Json) -> Result<()> { - let unsupported = - match protocol { - WireProtocol::OpenaiChat => { - response["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["message"].get("audio").is_some() - || choice["message"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => response["output"] - .as_array() - .into_iter() - .flatten() - .any(|item| { - matches!( - item.get("type").and_then(Json::as_str), - Some( - "reasoning" - | "computer_call" - | "computer_call_output" - | "web_search_call" - ) - ) - }), - WireProtocol::AnthropicMessages => response["content"] - .as_array() - .into_iter() - .flatten() - .any(|block| { - !matches!( - block.get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - }), + fn response_fixture(protocol: WireProtocol, extension: bool) -> Json { + let mut body = match protocol { + WireProtocol::OpenaiChat => json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": "provider/model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "world"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10} + }), + WireProtocol::OpenaiResponses => json!({ + "id": "resp-test", + "object": "response", + "model": "provider/model", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "world"}] + }], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10} + }), + WireProtocol::AnthropicMessages => json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": "provider/model", + "content": [{"type": "text", "text": "world"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 7, "output_tokens": 3} + }), }; - if unsupported { - Err(FlowError::InvalidArgument( - "provider-specific response extension cannot be translated safely".into(), - )) - } else { - Ok(()) + if extension { + body["provider_extension"] = json!({"exact": true}); + } + body } } diff --git a/crates/switchyard/tests/contract.rs b/crates/switchyard/tests/contract.rs deleted file mode 100644 index 7051f6a56..000000000 --- a/crates/switchyard/tests/contract.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for the public Switchyard Decision API contract. - -use nemo_relay_switchyard::contract::{DecisionAttempt, RoutingDecision}; -use serde_json::json; - -#[test] -fn current_switchyard_decision_contract_accepts_additive_fields() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"}, - "route": { - "tier": "capable", - "target_model": "model-a", - "backend_id": "backend-a", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.8, - "future_field": {"safe": true} - })) - .unwrap(); - assert_eq!( - decision.extra.get("future_field"), - Some(&json!({"safe": true})) - ); - assert!(decision.baseline_route.is_none()); -} - -#[test] -fn current_switchyard_decision_contract_accepts_an_explicit_baseline() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage", "version": "1"}, - "route": { - "tier": "efficient", - "target_model": "model-small", - "backend_id": "backend-small", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "baseline_route": { - "tier": "capable", - "target_model": "model-large", - "backend_id": "backend-large", - "target_protocol_profile": "anthropic_messages", - "target_endpoint": "/v1/messages" - } - })) - .unwrap(); - let baseline = decision.baseline_route.unwrap(); - assert_eq!(baseline.backend_id, "backend-large"); - assert_eq!(baseline.target_protocol_profile, "anthropic_messages"); -} - -#[test] -fn malformed_decisions_are_rejected_by_serde() { - let missing_route = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"} - }); - assert!(serde_json::from_value::(missing_route).is_err()); -} - -#[test] -fn additive_retry_metadata_stays_optional_on_the_wire() { - let attempt: DecisionAttempt = serde_json::from_value(json!({ - "routing_attempt": 1, - "max_routing_attempts": 4 - })) - .unwrap(); - assert!(attempt.previous_route.is_none()); - assert!(attempt.retry_reason.is_none()); -} diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs index 670d9159b..a252de07b 100644 --- a/crates/switchyard/tests/unit/component_tests.rs +++ b/crates/switchyard/tests/unit/component_tests.rs @@ -1,1827 +1,578 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Unit tests for the Switchyard Relay plugin component. - -use std::future::Future; -use std::pin::Pin; -use std::sync::Mutex; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::task::{Context, Poll}; +use std::sync::{Arc, Mutex}; -use axum::{ - Json as AxumJson, Router, - extract::State, - http::StatusCode, - routing::{get, post}, -}; -use futures_util::{Stream, stream as futures_stream}; -use nemo_relay::api::event::{Event, ScopeCategory}; -use nemo_relay::api::llm::{ - LlmCallExecuteParams, LlmStreamCallExecuteParams, llm_call_execute, llm_stream_call_execute, -}; -use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmStreamExecutionNextFn, LlmStreamInner}; -use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; -use nemo_relay::codec::optimization::LlmOptimizationSummaryStatus; -use nemo_relay::error::{UpstreamFailure, UpstreamFailureClass}; -use nemo_relay::plugin::rollback_registrations; +use futures_util::{StreamExt, stream}; +use nemo_relay::api::llm::LlmRequest; +use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmJsonStream, LlmStreamExecutionNextFn}; +use nemo_relay::error::{FlowError, UpstreamFailure, UpstreamFailureClass}; +use serde_json::{Map, Value as Json, json}; +use switchyard_libsy::{LlmResponse, Response as LibsyResponse, Step}; +use switchyard_protocol::text_response; use super::*; -struct CloseTrackingStream { - close_calls: Arc, - closed: bool, -} - -impl Stream for CloseTrackingStream { - type Item = FlowResult; - - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(None) - } -} - -impl LlmStreamInner for CloseTrackingStream { - fn close( - mut self: Pin<&mut Self>, - ) -> Pin> + Send + '_>> { - self.closed = true; - self.close_calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok(()) }) - } -} - -fn close_tracking_stream(close_calls: Arc) -> LlmJsonStream { - LlmJsonStream::from_closeable(CloseTrackingStream { - close_calls, - closed: false, - }) -} - -fn prefixed_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(json!({"first": true}))), - upstream, - }) -} - -fn terminal_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - mark_terminal_stream(upstream, "test", "enforce", json!({"route": "test"})) -} - -fn translated_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - upstream, - ) -} - -fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { +fn binding(protocol: WireProtocol, model: &str, weight: f64) -> TargetBinding { TargetBinding { model: model.into(), protocol, endpoint: protocol.endpoint().into(), - base_url: "http://127.0.0.1:9999".into(), + base_url: "https://provider.example.com/v1".into(), + weight, headers: BTreeMap::new(), header_env: BTreeMap::new(), } } -fn config(decision_api_url: String) -> SwitchyardConfig { +fn chat_config() -> SwitchyardConfig { SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets: BTreeMap::from([ - ( - "selected-chat".into(), - binding(WireProtocol::OpenaiChat, "selected"), - ), - ( - "baseline-chat".into(), - binding(WireProtocol::OpenaiChat, "baseline"), - ), - ( - "fallback-chat".into(), - binding(WireProtocol::OpenaiChat, "fallback"), - ), - ( - "fallback-responses".into(), - binding(WireProtocol::OpenaiResponses, "fallback"), - ), - ( - "fallback-anthropic".into(), - binding(WireProtocol::AnthropicMessages, "fallback"), - ), - ]), + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([( + "chat".into(), + binding(WireProtocol::OpenaiChat, "provider/chat", 1.0), + )]), default_targets: ProtocolDefaults { - openai_chat: "fallback-chat".into(), - openai_responses: "fallback-responses".into(), - anthropic_messages: "fallback-anthropic".into(), + openai_chat: "chat".into(), + ..ProtocolDefaults::default() }, + enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiChat]), ..SwitchyardConfig::default() } } -fn decision() -> RoutingDecision { - RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "decision-1".into(), - router: crate::contract::DecisionProvider { - name: "stage_router".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "selected".into(), - backend_id: "selected-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }, - baseline_route: Some(crate::contract::RoutingTarget { - tier: "capable".into(), - target_model: "baseline".into(), - backend_id: "baseline-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }), - confidence: Some(0.9), - reason_code: Some("test".into()), - reason_summary: None, - metadata: BTreeMap::from([ - ("feature_state".into(), json!("fresh")), - ("snapshot_age_millis".into(), json!(37)), - ("snapshot_max_age_millis".into(), json!(300_000)), - ]), - extra: BTreeMap::new(), - } -} - fn chat_request() -> LlmRequest { LlmRequest { - headers: Map::new(), + headers: Map::from_iter([ + ( + "x-nemo-relay-session-id".into(), + Json::String("session-1".into()), + ), + ( + "authorization".into(), + Json::String("Bearer caller-secret".into()), + ), + ]), content: json!({ - "model": "inbound", - "messages": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "answer"}, - {"role": "user", "content": "latest"} - ] + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"preserve": true} }), } } -fn request(protocol: WireProtocol) -> LlmRequest { - let content = match protocol { - WireProtocol::OpenaiChat => chat_request().content, - WireProtocol::OpenaiResponses => { - json!({"model": "inbound", "instructions": "system", "input": "latest"}) - } - WireProtocol::AnthropicMessages => { - json!({"model": "inbound", "system": "system", "max_tokens": 32, "messages": [{"role": "user", "content": "latest"}]}) +fn chat_response(model: &str, text: &str) -> Json { + json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": model, + "system_fingerprint": "fp_exact", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 } - }; - LlmRequest { - headers: Map::new(), - content, - } + }) } -fn chat_response() -> Json { +fn chat_chunk(model: &str, text: &str) -> Json { json!({ - "id": "chat-1", "object": "chat.completion", "model": "selected", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "model": model, + "system_fingerprint": "fp_stream_exact", + "choices": [{ + "index": 0, + "delta": {"content": text}, + "finish_reason": null + }] }) } -fn chat_chunk(text: &str, finish_reason: Json) -> Json { - json!({ - "id": "chat-1", "object": "chat.completion.chunk", "model": "selected", - "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": finish_reason}] +fn provider_error(class: UpstreamFailureClass, status: Option) -> FlowError { + FlowError::Upstream(UpstreamFailure { + status, + body: "provider failure".into(), + headers: BTreeMap::new(), + class, }) } #[test] -fn wire_protocol_and_plugin_lifecycle_contracts_are_stable() { - for (protocol, label, endpoint) in [ - ( - WireProtocol::OpenaiChat, - "openai_chat", - "/v1/chat/completions", - ), - ( - WireProtocol::OpenaiResponses, - "openai_responses", - "/v1/responses", - ), - ( - WireProtocol::AnthropicMessages, - "anthropic_messages", - "/v1/messages", - ), - ] { - assert_eq!(protocol.label(), label); - assert_eq!(protocol.endpoint(), endpoint); - assert_eq!( - WireProtocol::from_call(label, &request(protocol)), - Some(protocol) - ); - assert_eq!( - WireProtocol::from_call("unknown", &request(protocol)), - Some(protocol) - ); - } - assert_eq!( - WireProtocol::from_call( - "unknown", - &LlmRequest { - headers: Map::new(), - content: json!({}) - } - ), - None - ); - assert_eq!(RoutingMode::Enforce.label(), "enforce"); - assert_eq!(RoutingMode::ObserveOnly.label(), "observe_only"); - - let valid = config("http://127.0.0.1:1/v1/routing/decision".into()); - let component: PluginComponentSpec = valid.clone().into(); - assert_eq!(component.kind, SWITCHYARD_PLUGIN_KIND); - assert!(component.enabled); - assert_eq!(component.config["decision_profile_id"], "stage_router"); - - let plugin = SwitchyardPlugin; - assert_eq!(plugin.plugin_kind(), SWITCHYARD_PLUGIN_KIND); - assert!(!plugin.allows_multiple_components()); - assert!(plugin.validate(&component.config).is_empty()); - let diagnostics = plugin.validate(json!({"version": "invalid"}).as_object().unwrap()); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].level, DiagnosticLevel::Error); - assert_eq!(diagnostics[0].code, "switchyard.invalid_config"); +fn version_one_configuration_has_a_migration_error() { + let error = parse_config(&Map::from_iter([("version".into(), json!(1))])).unwrap_err(); + assert!(error.contains("removed switchyard-server Decision API")); + assert!(error.contains("version = 2")); } -#[tokio::test] -async fn plugin_registration_installs_and_rolls_back_both_execution_intercepts() { - let plugin = SwitchyardPlugin; - let decision_api_url = health_server(StatusCode::OK, r#"{"status":"ok"}"#).await; - for (mode, profile_id) in [ - (RoutingMode::Enforce, "random"), - (RoutingMode::Enforce, "llm"), - (RoutingMode::Enforce, "stage_router"), - (RoutingMode::ObserveOnly, "random"), - (RoutingMode::ObserveOnly, "llm"), - (RoutingMode::ObserveOnly, "stage_router"), - ] { - let mut candidate = config(decision_api_url.clone()); - candidate.mode = mode; - candidate.decision_profile_id = profile_id.into(); - let component: PluginComponentSpec = candidate.into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); +#[test] +fn random_configuration_rejects_invalid_targets_and_weights() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().endpoint = "/wrong".into(); + assert!(validate_config(&config).is_err()); - plugin - .register(&component.config, &mut context) - .await - .unwrap(); + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = f64::NAN; + assert!(validate_config(&config).is_err()); - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); - } + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + assert!(validate_config(&config).is_err()); } #[tokio::test] -async fn plugin_registration_retries_transient_sidecar_health_failures() { - let (decision_api_url, attempts) = recovering_health_server(2).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - plugin - .register(&component.config, &mut context) - .await +async fn run_stream_requires_relay_to_fulfill_the_provider_call() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let request = runtime + .libsy_request(WireProtocol::OpenaiChat, &chat_request(), false) .unwrap(); + let mut steps = runtime + .algorithm + .clone() + .run_stream(LibsyContext::default(), request); - assert_eq!(attempts.load(Ordering::SeqCst), 3); - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); -} + let decision = steps.next().await.unwrap().unwrap(); + assert!(matches!(decision, Step::Decision(_))); -#[tokio::test] -async fn plugin_registration_returns_the_final_health_failure_after_retry_exhaustion() { - let (decision_api_url, attempts) = recovering_health_server(usize::MAX).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert!( - error - .to_string() - .contains("returned HTTP 503 Service Unavailable") - ); - assert!(context.into_registrations().is_empty()); -} - -#[tokio::test] -async fn plugin_registration_rejects_unhealthy_or_invalid_sidecars() { - for (status, body, expected) in [ - ( - StatusCode::SERVICE_UNAVAILABLE, - r#"{"status":"starting"}"#, - "returned HTTP 503 Service Unavailable", - ), - (StatusCode::OK, "not-json", "returned invalid JSON"), - ( - StatusCode::OK, - r#"{"status":"starting"}"#, - "did not report status=ok", - ), - ] { - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(health_server(status, body).await).into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!( - error.to_string().contains(expected), - "{error:?} did not contain {expected:?}" - ); - assert!(context.into_registrations().is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_rejects_an_unreachable_sidecar() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = - config(format!("http://{address}/v1/routing/decision")).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!(error.to_string().contains("Switchyard service is required")); - assert!(error.to_string().contains("/health")); - assert!(context.into_registrations().is_empty()); -} - -#[test] -fn configuration_validation_rejects_unsafe_or_ambiguous_bindings() { - let url = "http://127.0.0.1:1/v1/routing/decision".to_string(); - let assert_invalid = |config: SwitchyardConfig, expected: &str| { - let error = SwitchyardRuntime::new(config) - .err() - .expect("config must fail"); - assert!( - error.contains(expected), - "{error:?} did not contain {expected:?}" - ); + let call = steps.next().await.unwrap().unwrap(); + let Step::CallLlm(call) = call else { + panic!("expected Relay-hosted CallLlm step"); }; - - let mut candidate = config(url.clone()); - candidate.version = 2; - assert_invalid(candidate, "unsupported Switchyard config version"); - let mut candidate = config(url.clone()); - candidate.decision_profile_id.clear(); - assert_invalid(candidate, "decision_profile_id"); - let mut candidate = config(url.clone()); - candidate.decision_timeout_millis = 0; - assert_invalid(candidate, "decision_timeout_millis"); - let mut candidate = config(url.clone()); - candidate.max_retries = 11; - assert_invalid(candidate, "max_retries"); - let mut candidate = config(url.clone()); - candidate.recent_message_count = 0; - assert_invalid(candidate, "recent_message_count"); - let mut candidate = config(url.clone()); - candidate.context_mode = ContextMode::AtofRequired; - assert_invalid(candidate, "atof_endpoint_name"); - let mut candidate = config(url.clone()); - candidate.atof_endpoint_name = Some(" switchyard ".into()); - assert_invalid(candidate, "leading or trailing whitespace"); - let candidate = config("file:///tmp/decision".into()); - assert_invalid(candidate, "must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.clear(); - assert_invalid(candidate, "targets must not be empty"); - let mut candidate = config(url.clone()); - candidate.enabled_inbound_profiles.clear(); - assert_invalid(candidate, "enabled_inbound_profiles"); - - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().endpoint = "/wrong".into(); - assert_invalid(candidate, "endpoint must be"); - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().base_url = "file:///tmp/provider".into(); - assert_invalid(candidate, "base_url must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.insert( - "duplicate-chat".into(), - candidate.targets["selected-chat"].clone(), - ); - assert_invalid(candidate, "conflicts with another exact backend binding"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "missing".into(); - assert_invalid(candidate, "default target \"missing\" is not configured"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "fallback-responses".into(); - assert_invalid(candidate, "must use protocol openai_chat"); - - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("bad header".into(), "value".into()); - assert_invalid(candidate, "invalid header name"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("x-test".into(), "bad\nvalue".into()); - assert_invalid(candidate, "invalid header value"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("Authorization".into(), "Bearer static".into()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url.clone()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); - let mut candidate = config(url.clone()); - let target = candidate.targets.get_mut("selected-chat").unwrap(); - target - .headers - .insert("Authorization".into(), "Bearer static".into()); - target.header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url); - candidate - .targets - .get_mut("selected-chat") - .unwrap() - .header_env - .insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); -} - -#[test] -fn disabled_protocol_defaults_are_optional() { - let defaults: ProtocolDefaults = serde_json::from_value(json!({ - "anthropic_messages": "fallback-anthropic" + assert_eq!(call.get_decision().selected_model(), "chat"); + call.respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response(Some("provider/chat".into()), "hello")), + metadata: None, })) - .expect("omitted disabled protocol defaults should deserialize"); - assert!(defaults.openai_chat.is_empty()); - assert!(defaults.openai_responses.is_empty()); - - let mut candidate = config("http://127.0.0.1:1/v1/routing/decision".into()); - candidate.enabled_inbound_profiles = BTreeSet::from([WireProtocol::AnthropicMessages]); - candidate.default_targets.openai_chat.clear(); - candidate.default_targets.openai_responses.clear(); - - SwitchyardRuntime::new(candidate).expect("Anthropic-only configuration should validate"); -} - -#[test] -fn atof_cross_component_validation_rejects_invalid_endpoint_names_before_lookup() { - for (name, expected) in [ - (" ", "atof_endpoint_name must be non-empty when configured"), - ( - " switchyard ", - "atof_endpoint_name must not have leading or trailing whitespace", - ), - ] { - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some(name.into()); - let plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; + .unwrap(); - assert_eq!( - validate_switchyard_atof_configuration(&plugin_config).unwrap_err(), - expected - ); - } + let returned = steps.next().await.unwrap().unwrap(); + assert!(matches!(returned, Step::ReturnToAgent(_))); } -#[test] -fn atof_cross_component_validation_reports_each_activation_mismatch() { - assert!(validate_switchyard_atof_configuration(&PluginConfig::default()).is_ok()); - - let payload = PluginConfig { - components: vec![config("http://switchyard.test/v1/routing/decision".into()).into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&payload).is_ok()); - - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": {"enabled": true, "sinks": [{ - "type": "stream", - "name": "other", - "url": "http://events.test/v1/atof/events", - "header_env": {"authorization": "TOKEN"} - }]}}) - .as_object() - .unwrap() - .clone(), +#[tokio::test] +async fn buffered_dispatch_uses_relay_binding_and_preserves_same_protocol_json() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let captured = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&captured); + let expected = chat_response("provider/chat", "routed"); + let provider_response = expected.clone(); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + let response = provider_response.clone(); + Box::pin(async move { + capture.lock().unwrap().push(request); + Ok(response) + }) }); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("requires named ATOF endpoint") - ); - plugin_config.components[1].config["atof"]["sinks"][0]["name"] = json!("switchyard"); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("websocket"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("transport = http_post") - ); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("http_post"); - let duplicate = plugin_config.components[1].config["atof"]["sinks"][0].clone(); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .push(duplicate); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("exactly one endpoint") - ); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .pop(); - plugin_config.components[1].config["atof"]["sinks"][0]["field_name_policy"] = - json!("snake_case"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("field_name_policy = preserve") - ); - let endpoint = &mut plugin_config.components[1].config["atof"]["sinks"][0]; - endpoint["field_name_policy"] = json!("preserve"); - endpoint.as_object_mut().unwrap().remove("header_env"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("environment-referenced header") - ); -} + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + assert_eq!(response, expected); -#[tokio::test] -async fn execution_bypasses_inapplicable_calls_and_fails_open_on_extensions() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let passthrough: LlmExecutionNextFn = - Arc::new(|request| Box::pin(async move { Ok(request.content) })); - let unknown = LlmRequest { - headers: Map::new(), - content: json!({"opaque": true}), - }; + let requests = captured.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].content["model"], "provider/chat"); + assert_eq!(requests[0].content["stream"], false); assert_eq!( - runtime - .execute_buffered("custom.provider", unknown, Arc::clone(&passthrough)) - .await - .unwrap()["opaque"], - true + requests[0].content["provider_extension"], + json!({"preserve": true}) ); - - let mut disabled_config = config("http://127.0.0.1:1/v1/routing/decision".into()); - disabled_config - .enabled_inbound_profiles - .remove(&WireProtocol::OpenaiChat); - let disabled = SwitchyardRuntime::new(disabled_config).unwrap(); assert_eq!( - disabled - .execute_buffered( - "openai.chat_completions", - chat_request(), - Arc::clone(&passthrough), - ) - .await - .unwrap()["model"], - "inbound" + requests[0].headers[INTERNAL_DISPATCH_URL_HEADER], + "https://provider.example.com/v1/chat/completions" ); - - let mut unsupported = chat_request(); - unsupported.content["thinking"] = json!({"type": "enabled"}); - let response = runtime - .execute_buffered( - "openai.chat_completions", - unsupported, - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); + assert!(!requests[0].headers.contains_key("authorization")); + assert!(!requests[0].headers.contains_key("x-nemo-relay-session-id")); } #[tokio::test] -async fn stream_setup_retries_and_empty_streams_have_one_bounded_fallback() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connect".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })); - } - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); - - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); +async fn cross_protocol_buffered_route_uses_switchyard_translation_both_ways() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + config.targets.insert( + "anthropic".into(), + binding(WireProtocol::AnthropicMessages, "provider/anthropic", 1.0), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmExecutionNextFn = Arc::new(move |request| { Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt < 4 { - Vec::new() - } else { - assert_eq!(request.content["model"], "fallback"); - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) + assert_eq!(request.content["model"], "provider/anthropic"); + assert_eq!(request.content["messages"][0]["role"], "user"); + assert!(request.content.get("input").is_none()); + Ok(json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": "provider/anthropic", + "content": [{"type": "text", "text": "translated"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 1} + })) }) }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - assert_eq!(decisions.lock().unwrap().len(), 4); -} - -#[tokio::test] -async fn fallback_setup_failures_preserve_the_provider_error() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let buffered: LlmExecutionNextFn = Arc::new(|_| { - Box::pin(async { Err(FlowError::Internal("buffered fallback failed".into())) }) - }); - let error = runtime - .dispatch_fallback_buffered(WireProtocol::OpenaiChat, chat_request(), buffered, "test") - .await - .unwrap_err(); - assert!(error.to_string().contains("buffered fallback failed")); - let streaming: LlmStreamExecutionNextFn = - Arc::new(|_| Box::pin(async { Err(FlowError::Internal("stream fallback failed".into())) })); - let error = runtime - .dispatch_fallback_stream(WireProtocol::OpenaiChat, chat_request(), streaming, "test") + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) .await - .err() - .expect("stream fallback setup must fail"); - assert!(error.to_string().contains("stream fallback failed")); -} - -#[tokio::test] -async fn translated_stream_preserves_success_and_propagates_both_error_sources() { - let source = LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("hello", Json::Null)), - Ok(chat_chunk("", json!("stop"))), - ])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output.iter().all(Result::is_ok)); - assert!(output.iter().any(|item| { - item.as_ref() - .is_ok_and(|chunk| chunk.to_string().contains("hello")) - })); - - let upstream_error = FlowError::Internal("upstream stream failed".into()); - let source = LlmJsonStream::new(futures_stream::iter(vec![Err(upstream_error)])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output[0].is_err()); - - let malformed = LlmJsonStream::new(futures_stream::iter(vec![Ok(json!({ - "choices": [{"delta": {"reasoning_content": "private"}}] - }))])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - malformed, - ) - .collect::>() - .await; - assert!(output[0].is_err()); + .unwrap(); + assert_eq!(response["choices"][0]["message"]["content"], "translated"); } #[tokio::test] -async fn stream_adapters_forward_explicit_close_to_the_upstream_stream() { - for make_adapter in [ - prefixed_adapter as fn(LlmJsonStream) -> LlmJsonStream, - terminal_adapter, - translated_adapter, - ] { - let close_calls = Arc::new(AtomicUsize::new(0)); - let mut stream = make_adapter(close_tracking_stream(Arc::clone(&close_calls))); - - stream.close().await.unwrap(); - - assert_eq!(close_calls.load(Ordering::SeqCst), 1); - assert!(stream.next().await.is_none()); +async fn deterministic_seed_reproduces_the_same_target_sequence() { + fn config() -> SwitchyardConfig { + let mut config = chat_config(); + config.targets.insert( + "other".into(), + binding(WireProtocol::OpenaiChat, "provider/other", 1.0), + ); + config } -} -#[test] -fn provider_failure_reporting_covers_every_retry_class() { - for (class, label, retryable) in [ - (UpstreamFailureClass::Connection, "connection", true), - (UpstreamFailureClass::Timeout, "timeout", true), - ( - UpstreamFailureClass::RetryableStatus, - "retryable_status", - true, - ), - (UpstreamFailureClass::ContextWindow, "context_window", true), - ( - UpstreamFailureClass::ModelUnavailable, - "model_unavailable", - true, - ), - ( - UpstreamFailureClass::Authentication, - "authentication", - false, - ), - ( - UpstreamFailureClass::InvalidRequest, - "invalid_request", - false, - ), - (UpstreamFailureClass::Other, "other", false), - ] { - let error = FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "failure".into(), - headers: BTreeMap::new(), - class, + async fn sequence(runtime: &SwitchyardRuntime) -> Vec { + let captured = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&captured); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + capture.lock().unwrap().push(model.clone()); + Ok(chat_response(&model, "ok")) + }) }); - assert_eq!(provider_error_class(&error), label); - assert_eq!(error_is_retryable(&error), retryable); - assert_eq!(provider_error_summary(&error), format!("{label}:http_503")); - } - let relay = FlowError::Internal("failure".into()); - assert_eq!(provider_error_class(&relay), "relay"); - assert_eq!(provider_error_summary(&relay), relay.to_string()); -} - -#[test] -fn all_materialization_modes_are_bounded_and_provider_valid() { - for protocol in [ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ] { - for mode in [ - RequestMaterialization::None, - RequestMaterialization::SummaryOnly, - RequestMaterialization::LatestUserPrompt, - RequestMaterialization::RecentMessageWindow, - RequestMaterialization::AnnotatedRequest, - RequestMaterialization::FullBody, - ] { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - config.request_materialization = mode; - config.recent_message_count = 2; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let routing = runtime - .routing_request(protocol, &request(protocol), 1, None) + for _ in 0..12 { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next.clone()) + .await .unwrap(); - match mode { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => { - assert!(routing.current_request.is_none()) - } - RequestMaterialization::LatestUserPrompt => { - let current = routing.current_request.unwrap(); - assert_eq!(current["latest_user_prompt"], "latest"); - let body = current["body"].clone(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: body, - }, - ) - .unwrap(); - } - RequestMaterialization::RecentMessageWindow => { - let current = routing.current_request.unwrap(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: current["body"].clone(), - }, - ) - .unwrap(); - } - RequestMaterialization::AnnotatedRequest | RequestMaterialization::FullBody => { - assert!(routing.current_request.is_some()) - } - } } + captured.lock().unwrap().clone() } -} - -#[test] -fn identity_policy_requires_stable_request_scope_only_for_atof_profiles() { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - let payload_runtime = SwitchyardRuntime::new(config.clone()).unwrap(); - let synthetic = payload_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - assert_eq!(synthetic.identity.quality, "synthetic"); - config.context_mode = ContextMode::AtofRequired; - config.atof_endpoint_name = Some("switchyard".into()); - let atof_runtime = SwitchyardRuntime::new(config).unwrap(); - assert!( - atof_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .is_err() - ); - let mut stable = chat_request(); - stable - .headers - .insert("x-nemo-relay-session-id".into(), json!("session-1")); - stable - .headers - .insert("x-nemo-relay-request-id".into(), json!("request-1")); - let routed = atof_runtime - .routing_request(WireProtocol::OpenaiChat, &stable, 1, None) - .unwrap(); - assert_eq!(routed.identity.quality, "explicit"); + let first = SwitchyardRuntime::new(config()).unwrap(); + let second = SwitchyardRuntime::new(config()).unwrap(); + let first_sequence = sequence(&first).await; + let second_sequence = sequence(&second).await; + assert_eq!(first_sequence, second_sequence); + assert!(first_sequence.contains(&"provider/chat".to_string())); + assert!(first_sequence.contains(&"provider/other".to_string())); } -#[test] -fn exact_target_validation_rejects_any_switchyard_drift() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - assert!(runtime.validate_decision(&decision()).is_ok()); - let mut drifted = decision(); - drifted.route.target_model = "unbound-model".into(); - assert!(runtime.validate_decision(&drifted).is_err()); - drifted = decision(); - drifted.route.backend_id = "unknown".into(); - assert!(runtime.validate_decision(&drifted).is_err()); -} - -#[test] -fn routing_contribution_requires_an_exact_independent_baseline_binding() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - - let contribution = runtime.routing_contribution(&decision(), 2, true).unwrap(); - assert!(contribution.applied); - assert_eq!( - contribution.kind.as_str(), - LlmOptimizationKind::MODEL_ROUTING - ); - let transition = contribution.model_transition.unwrap(); - assert_eq!(transition.baseline.unwrap().model, "baseline"); - assert_eq!(transition.effective.unwrap().model, "selected"); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["feature_state"], - "fresh" - ); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["snapshot_age_millis"], - 37 - ); - assert_eq!( - contribution.payload_schema.as_ref().unwrap().name, - ROUTING_CONTRIBUTION_SCHEMA - ); - - let observed = runtime.routing_contribution(&decision(), 1, false).unwrap(); - assert!(!observed.applied); - - let mut missing = decision(); - missing.baseline_route = None; - assert!(runtime.routing_contribution(&missing, 1, true).is_none()); - - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - assert!(runtime.routing_contribution(&drifted, 1, true).is_none()); - assert!(runtime.validate_decision(&drifted).is_ok()); -} - -#[test] -fn routing_decision_mark_has_canonical_shape_and_mirrored_identity() { - let subscriber_name = format!("switchyard-mark-shape-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let routing_request = runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - runtime.emit_decision(&routing_request, &decision(), 1, false, 17); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - - let event = events - .lock() - .unwrap() - .iter() - .map(Event::to_json_value) - .find(|event| { - event["name"] == "switchyard.routing.decision" - && event["metadata"]["session_id"] == routing_request.identity.session_id - && event["metadata"]["request_id"] == routing_request.identity.request_id +#[tokio::test] +async fn retryable_provider_failure_starts_a_fresh_libsy_run() { + let mut config = chat_config(); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + Box::pin(async move { + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error( + UpstreamFailureClass::RetryableStatus, + Some(503), + )); + } + Ok(chat_response( + request.content["model"].as_str().unwrap(), + "retried", + )) }) - .expect("decision mark should be captured"); - assert_eq!(event["kind"], "mark"); - assert_eq!(event["category"], "custom"); - assert_eq!( - event["category_profile"]["subtype"], - "switchyard.routing.decision" - ); - assert_eq!(event["data_schema"]["name"], ROUTING_MARK_SCHEMA); - assert_eq!(event["data_schema"]["version"], "1"); - assert_eq!(event["data"]["profile_id"], "stage_router"); - assert_eq!(event["data"]["selected_model"], "selected"); - assert_eq!(event["data"]["latency_ms"], 17); - assert_eq!(event["data"]["router_metadata"]["feature_state"], "fresh"); - assert_eq!(event["data"]["router_metadata"]["snapshot_age_millis"], 37); - assert_eq!( - event["metadata"]["session_id"], - routing_request.identity.session_id - ); - assert_eq!( - event["metadata"]["request_id"], - routing_request.identity.request_id - ); -} - -#[test] -fn atof_required_cross_component_validation_is_context_sensitive() { - let mut switchyard = config("http://switchyard.test:8080/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&plugin_config).is_err()); - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": { - "enabled": true, - "sinks": [{ - "type": "stream", - "name": "switchyard", - "url": "http://switchyard.test:8080/v1/atof/events", - "transport": "http_post", - "field_name_policy": "preserve", - "header_env": {"authorization": "SWITCHYARD_TOKEN"} - }] - }}) - .as_object() - .unwrap() - .clone(), }); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); -} - -#[derive(Clone)] -struct DecisionState { - requests: Arc>>, - decision: RoutingDecision, -} - -async fn decision_handler( - State(state): State, - AxumJson(request): AxumJson, -) -> AxumJson { - state.requests.lock().unwrap().push(request); - AxumJson(state.decision) -} - -async fn decision_server() -> (String, Arc>>) { - decision_server_for(decision()).await -} - -async fn decision_server_for( - decision: RoutingDecision, -) -> (String, Arc>>) { - let requests = Arc::new(Mutex::new(Vec::new())); - let state = DecisionState { - requests: Arc::clone(&requests), - decision, - }; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new() - .route("/v1/routing/decision", post(decision_handler)) - .with_state(state), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), requests) -} - -async fn health_server(status: StatusCode, body: &'static str) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || async move { (status, [("content-type", "application/json")], body) }), - ), - ) - .await - .unwrap(); - }); - format!("http://{address}/nested/v1/routing/decision?source=test#ignored") -} -async fn recovering_health_server(failures_before_ready: usize) -> (String, Arc) { - let attempts = Arc::new(AtomicUsize::new(0)); - let captured = Arc::clone(&attempts); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || { - let attempts = Arc::clone(&captured); - async move { - if attempts.fetch_add(1, Ordering::SeqCst) < failures_before_ready { - ( - StatusCode::SERVICE_UNAVAILABLE, - AxumJson(json!({"status": "starting"})), - ) - } else { - (StatusCode::OK, AxumJson(json!({"status": "ok"}))) - } - } - }), - ), - ) + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) .await .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), attempts) -} - -async fn managed_buffered_events( - runtime: SwitchyardRuntime, - next: LlmExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_buffered("openai.chat_completions", request, next) - .await - }) - }); - llm_call_execute( - LlmCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .build(), - ) - .await - .unwrap(); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} - -async fn managed_stream_events( - runtime: SwitchyardRuntime, - next: LlmStreamExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-stream-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmStreamExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); - Box::pin(async move { - runtime - .execute_stream("openai.chat_completions", request, next) - .await - }) - }); - let mut stream = llm_stream_call_execute( - LlmStreamCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .collector(Box::new(|_| Ok(()))) - .finalizer(Box::new(|| json!({"done": true}))) - .build(), - ) - .await - .unwrap(); - while stream.next().await.is_some() {} - drop(stream); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() + assert_eq!(response["choices"][0]["message"]["content"], "retried"); + assert_eq!(calls.load(Ordering::SeqCst), 2); } #[tokio::test] -async fn buffered_accounting_records_only_the_terminal_committed_route() { - let (url, _) = decision_server().await; - let successful = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| Box::pin(async { Ok(chat_response()) })), - ) - .await; - let marks = successful - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].data().unwrap()["applied"], true); - assert_eq!( - marks[0].data().unwrap()["model_transition"]["baseline"]["model"], - "baseline" - ); - let summary = successful - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") +async fn non_retryable_failure_dispatches_trusted_fallback_exactly_once() { + let mut config = chat_config(); + config.targets.insert( + "fallback".into(), + binding(WireProtocol::OpenaiChat, "provider/fallback", 0.0), ); -} - -#[tokio::test] -async fn retry_then_success_records_one_terminal_route_with_matching_mark_identity() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry once".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert_eq!(requests[1].attempt.routing_attempt, 2); - drop(requests); - - let start = events - .iter() - .find(|event| { - event.name() == "openai.chat_completions" - && event.scope_category() == Some(ScopeCategory::Start) + config.default_targets.openai_chat = "fallback".into(); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + capture.lock().unwrap().push(model.clone()); + if model == "provider/chat" { + return Err(provider_error( + UpstreamFailureClass::InvalidRequest, + Some(400), + )); + } + Ok(chat_response(&model, "fallback")) }) - .unwrap(); - let marks = events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].parent_uuid(), Some(start.uuid())); - assert_eq!(marks[0].data().unwrap()["payload"]["routing_attempt"], 2); + }); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await .unwrap(); - assert_eq!(summary.contributions.len(), 1); - let contribution = &summary.contributions[0]; - assert!(contribution.applied); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!(marks[0].data().unwrap()["id"], json!(contribution.id)); + assert_eq!(response["choices"][0]["message"]["content"], "fallback"); assert_eq!( - marks[0].data().unwrap()["sequence"], - json!(contribution.sequence) + *models.lock().unwrap(), + vec!["provider/chat", "provider/fallback"] ); } #[tokio::test] -async fn oversized_decision_metadata_limits_accounting_without_failing_provider_success() { - let mut oversized = decision(); - oversized - .metadata - .insert("oversized_evidence".into(), json!("x".repeat(20_000))); - let (url, _) = decision_server_for(oversized).await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); +async fn concurrent_runs_dispatch_independently() { + let runtime = Arc::new(SwitchyardRuntime::new(chat_config()).unwrap()); + let calls = Arc::new(AtomicUsize::new(0)); + let next: LlmExecutionNextFn = { + let calls = Arc::clone(&calls); + Arc::new(move |request| { + let calls = Arc::clone(&calls); Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(chat_response()) + calls.fetch_add(1, Ordering::SeqCst); + Ok(chat_response( + request.content["model"].as_str().unwrap(), + "ok", + )) }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.status, LlmOptimizationSummaryStatus::Partial); - assert!(summary.contributions.is_empty()); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "contribution_limit_exceeded") - ); -} - -#[tokio::test] -async fn malformed_baseline_is_telemetry_only_and_does_not_change_the_selected_route() { - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - let (url, _) = decision_server_for(drifted).await; - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "selected"); - Ok(chat_response()) - }) - }), - ) - .await; - assert!(events.iter().any(|event| { - event.name() == "switchyard.routing.error" - && event.data().is_some_and(|data| { - data["error_class"] == "baseline_binding" - && data["error"] - .as_str() - .is_some_and(|error| error.contains("exact Relay binding")) - }) - })); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn observe_only_accounting_is_visible_but_not_applied() { - let (url, _) = decision_server().await; - let mut observe = config(url); - observe.mode = RoutingMode::ObserveOnly; - let events = managed_buffered_events( - SwitchyardRuntime::new(observe).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await; - let contribution = events - .iter() - .find(|event| event.name() == "nemo_relay.llm.optimization") - .and_then(Event::data) - .unwrap(); - assert_eq!(contribution["applied"], false); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(!summary.contributions[0].applied); - assert!(summary.tokens_saved.total_tokens.is_none()); -} - -#[tokio::test] -async fn streaming_accounting_commits_on_the_first_successful_item_only() { - let (url, _) = decision_server().await; - let committed = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| { - Box::pin(async { - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }), - ) - .await; - assert_eq!( - committed - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let items = if seen.fetch_add(1, Ordering::SeqCst) == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - }))] - } else { - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); -} - -#[tokio::test] -async fn committed_stream_error_keeps_one_route_and_never_redispatches() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed after commit".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]))) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); - assert_eq!( - events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 - ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) + }; + let runs = (0..16).map(|_| { + let runtime = Arc::clone(&runtime); + let next = next.clone(); + tokio::spawn(async move { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "stream_interrupted") - ); + }); + for result in futures_util::future::join_all(runs).await { + assert!(result.unwrap().is_ok()); + } + assert_eq!(calls.load(Ordering::SeqCst), 16); } #[tokio::test] -async fn retry_exhaustion_redecides_four_times_then_dispatches_fallback_once() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); +async fn streaming_prefetch_retries_before_commit() { + let mut config = chat_config(); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let capture = Arc::clone(&capture); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst) + 1; - if attempt <= 4 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "temporarily unavailable".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })); + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error(UpstreamFailureClass::Connection, None)); } - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + "provider/chat", + "retried", + ))]))) }) }); - let response = runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 4); - assert_eq!(requests[3].attempt.routing_attempt, 4); - assert_eq!( - requests[3].attempt.previous_route.as_deref(), - Some("selected-chat") - ); + .unwrap() + .collect::>() + .await; + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert!(output.iter().all(Result::is_ok)); } #[tokio::test] -async fn non_retryable_provider_failure_bypasses_retry_loop() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); +async fn first_provider_error_event_retries_before_commit() { + let mut config = chat_config(); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let capture = Arc::clone(&capture); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - if attempt == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })); + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Ok(LlmJsonStream::new(stream::iter([Ok(json!({ + "error": {"message": "temporary provider failure"} + }))]))); } - Ok(chat_response()) + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + "provider/chat", + "retried", + ))]))) }) }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 1); + .unwrap() + .collect::>() + .await; + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert!(output.iter().all(Result::is_ok)); } #[tokio::test] -async fn observe_only_records_one_decision_and_dispatches_only_the_trusted_default() { - let (url, decisions) = decision_server().await; - let mut config = config(url); - config.mode = RoutingMode::ObserveOnly; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); +async fn streaming_late_error_is_propagated_without_retry() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let capture = Arc::clone(&capture); Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) + capture.fetch_add(1, Ordering::SeqCst); + Ok(LlmJsonStream::new(stream::iter([ + Ok(chat_chunk("provider/chat", "partial")), + Err(provider_error(UpstreamFailureClass::Connection, None)), + ]))) }) }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); + .unwrap() + .collect::>() + .await; + assert!(output.first().is_some_and(Result::is_ok)); + assert!(output.last().is_some_and(Result::is_err)); + assert_eq!(calls.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn streaming_retries_before_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); +async fn late_provider_error_event_is_propagated_without_retry() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); + let capture = Arc::clone(&capture); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - }))] - } else { - vec![Ok(chat_chunk("ok", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) + capture.fetch_add(1, Ordering::SeqCst); + Ok(LlmJsonStream::new(stream::iter([ + Ok(chat_chunk("provider/chat", "partial")), + Ok(json!({"error": {"message": "late provider failure"}})), + ]))) }) }); - let stream = runtime + + let output = runtime .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 1); - assert!(output[0].is_ok()); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); + .unwrap() + .collect::>() + .await; + assert!(output.first().is_some_and(Result::is_ok)); + assert!(output.last().is_some_and(Result::is_err)); + assert_eq!(calls.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn streaming_never_retries_after_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); +async fn cross_protocol_stream_uses_switchyard_translation_both_ways() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + config.targets.insert( + "anthropic".into(), + binding(WireProtocol::AnthropicMessages, "provider/anthropic", 1.0), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - let items = vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, + assert_eq!(request.content["model"], "provider/anthropic"); + Ok(LlmJsonStream::new(stream::iter([ + Ok(json!({ + "type": "message_start", + "message": { + "id": "msg-stream", + "type": "message", + "role": "assistant", + "model": "provider/anthropic", + "content": [], + "usage": {"input_tokens": 2, "output_tokens": 0} + } + })), + Ok(json!({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""} + })), + Ok(json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "translated"} + })), + Ok(json!({"type": "content_block_stop", "index": 0})), + Ok(json!({ + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 1} })), - ]; - Ok(LlmJsonStream::new(futures_stream::iter(items))) + Ok(json!({"type": "message_stop"})), + ]))) }) }); - let stream = runtime + + let output = runtime .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 2); - assert!(output[0].is_ok()); - assert!(output[1].is_err()); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); -} - -fn responses_config(decision_api_url: String) -> SwitchyardConfig { - let targets = BTreeMap::from([ - ( - "resp-a".into(), - binding(WireProtocol::OpenaiResponses, "model-a"), - ), - ( - "resp-b".into(), - binding(WireProtocol::OpenaiResponses, "model-b"), - ), - ]); - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets, - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: "resp-a".into(), - anthropic_messages: String::new(), - }, - enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiResponses]), - ..SwitchyardConfig::default() - } -} - -fn responses_decision() -> RoutingDecision { - RoutingDecision { - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "model-b".into(), - backend_id: "resp-b".into(), - target_protocol_profile: "openai_responses".into(), - target_endpoint: "/v1/responses".into(), - }, - baseline_route: None, - ..decision() - } + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output + .iter() + .filter_map(|result| result.as_ref().ok()) + .any(|event| { + event["choices"][0]["delta"]["content"] + .as_str() + .is_some_and(|text| text == "translated") + }) + ); } -// When every configured target shares the inbound protocol, no cross-protocol translation can -// occur, so the portability guard is skipped: a non-portable Responses request (real Codex -// extensions) is routed through the Decision API with its provider-specific fields preserved. #[tokio::test] -async fn same_protocol_targets_route_nonportable_streaming_requests() { - let (url, decisions) = decision_server_for(responses_decision()).await; - let runtime = SwitchyardRuntime::new(responses_config(url)).unwrap(); - let mut nonportable = request(WireProtocol::OpenaiResponses); - nonportable.content["reasoning"] = json!({"effort": "high"}); - nonportable.content["store"] = json!(true); - let next: LlmStreamExecutionNextFn = Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "model-b"); - assert_eq!(request.content["store"], true); - assert_eq!(request.content["reasoning"]["effort"], "high"); - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - json!({"type": "response.output_text.delta", "delta": "ok"}), - )]))) - }) +async fn same_protocol_stream_round_trip_preserves_raw_provider_event() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let raw = chat_chunk("provider/chat", "exact"); + let provider_chunk = raw.clone(); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let chunk = provider_chunk.clone(); + Box::pin(async move { Ok(LlmJsonStream::new(stream::iter([Ok(chunk)]))) }) }); + let output = runtime - .execute_stream("openai.responses", nonportable, next) + .execute_stream("openai.chat_completions", chat_request(), next) .await .unwrap() .collect::>() .await; assert_eq!(output.len(), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); + assert_eq!(output[0].as_ref().ok(), Some(&raw)); +} + +#[test] +fn plugin_source_does_not_use_relay_translation_codecs() { + let component = include_str!("../../src/component.rs"); + let translation = include_str!("../../src/translation.rs"); + let stream_translation = include_str!("../../src/stream_translation.rs"); + for source in [component, translation, stream_translation] { + assert!(!source.contains("nemo_relay::codec")); + } } diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index 054e11189..ec52646ae 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -34,8 +34,8 @@ entries in `plugins.toml`. - [PII Redaction](/configure-plugins/pii-redaction/about) sanitizes sensitive data in observability payloads. - [Switchyard (Experimental)](/configure-plugins/switchyard/about) routes - requests using decisions from a separately running Switchyard Decision API - service. + requests in process through Switchyard libsy while Relay dispatches provider + calls. - [Model Pricing](/configure-plugins/model-pricing) configures catalog sources for cost estimates on managed LLM responses. diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx index 321a4d43d..86f2093cd 100644 --- a/docs/configure-plugins/switchyard/about.mdx +++ b/docs/configure-plugins/switchyard/about.mdx @@ -1,7 +1,7 @@ --- title: "Switchyard (Experimental)" sidebar-title: "Switchyard (Experimental)" -description: "Set up and validate the experimental Switchyard Decision API integration for NeMo Relay 0.6.0." +description: "Route LLM calls in process with Switchyard libsy." position: 5 --- import { MermaidStyles } from "@/components/MermaidStyles"; @@ -9,196 +9,64 @@ import { MermaidStyles } from "@/components/MermaidStyles"; {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -> **Experimental:** The Switchyard integration is an early-access feature. It -> is not enabled in default Relay builds, its configuration and contracts can -> change, and the current deployment requires a separately running Switchyard -> Decision API service. +> **Experimental:** The Switchyard component is compile-time optional and its +> library contract may change before the compatible Switchyard 0.2 crates are +> published. -> **NeMo Relay 0.6.0 architecture:** This release uses the external Switchyard -> Decision API from the -> [`topic/nemo-relay-integration`](https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration) -> branch. +The Switchyard component runs libsy's random router inside NeMo Relay. Relay +drives `Algorithm::run_stream`, performs every provider call, and sends the real +response or error back through `CallLlmRequest::respond`. -Switchyard is an LLM routing decision engine. The `nemo-relay-switchyard` -plugin asks Switchyard which configured target should handle an LLM request. -Relay validates the decision and performs the authenticated provider request. -For example, a routing profile can send a simple prompt to a lower-cost model -and a complex prompt to a more capable model. +There is no `switchyard-server`, Decision API, service health check, Switchyard +LLM client, or dedicated Switchyard ATOF receiver. ## Architecture -The following diagram shows the NeMo Relay 0.6.0 service boundary: - ```mermaid flowchart LR - Agent["Agent"] -->|LLM request| Relay["NeMo Relay"] - Relay -->|routing request| Decision["Switchyard Decision API"] - Decision -->|backend decision| Relay - Relay -->|authenticated provider request| Backend["Selected LLM backend"] - Relay -->|ATOF events| History["Switchyard ATOF accumulator"] - History -. routing history .-> Decision + Agent["Agent request"] --> Decode["switchyard-translation decode"] + Decode --> Libsy["libsy run_stream"] + Libsy --> Call["CallLlm"] + Call --> EncodeTarget["switchyard-translation encode target"] + EncodeTarget --> Relay["Relay provider dispatch"] + Relay --> DecodeResponse["switchyard-translation decode response"] + DecodeResponse --> Respond["CallLlmRequest.respond"] + Respond --> Final["ReturnToAgent"] + Final --> EncodeCaller["switchyard-translation encode caller"] + EncodeCaller --> Agent class Agent grey-lightest; class Relay green-lightest; - class Decision yellow-lightest; - class History yellow-lightest; - class Backend teal-lightest; + class Libsy yellow-lightest; + class Decode,EncodeTarget,DecodeResponse,EncodeCaller teal-lightest; ``` -The components divide responsibility as follows: - -| Component | Responsibilities | -| --- | --- | -| Switchyard | Selects a backend and accumulates ATOF routing history for history-based profiles. | -| NeMo Relay | Owns provider credentials, target bindings, decision validation, protocol translation, dispatch, retries, trusted fallback, and observability. | - -Relay does not start or supervise the Switchyard service. Start Switchyard -before Relay activates the plugin. Relay derives the service's root `/health` -URL from `decision_api_url` and fails activation unless the endpoint returns -`{"status":"ok"}`. +## Responsibility Boundary -## Prerequisites - -Install the following prerequisites before you start: - -| Requirement | Version or Value | +| Owner | Responsibilities | | --- | --- | -| NeMo Relay | Tag `0.6.0` | -| Rust | `1.96.1` | -| Switchyard branch | `topic/nemo-relay-integration` | -| Switchyard commit | `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8` | -| Local commands | `git`, `python3`, and `curl` | - -Keep the repositories in the following sibling layout. The validation script -uses this layout by default: - -```text -/ -├── NeMo-Relay/ -└── Switchyard-topic-nemo-relay-integration/ -``` - -## Build and Run the Compatibility Test - -Use the following procedure to clone and pin both repositories: - -```bash -mkdir relay-switchyard-0.6 -cd relay-switchyard-0.6 - -git clone --branch 0.6.0 \ - https://github.com/NVIDIA/NeMo-Relay.git \ - NeMo-Relay - -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - Switchyard-topic-nemo-relay-integration - -git -C Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 - -cd NeMo-Relay -``` +| libsy | Random target selection, decision emission, algorithm lifecycle, and consuming the response supplied by Relay. | +| `switchyard-translation` | Decode and encode OpenAI Chat, OpenAI Responses, and Anthropic Messages requests, buffered responses, and stream events. | +| NeMo Relay | Provider bindings, credentials, URL and endpoint selection, physical dispatch, retries, trusted fallback, scopes, and observability. | -The Switchyard plugin is excluded from default Relay CLI builds. Build the CLI -with the optional feature explicitly: +Translation also runs for same-protocol calls because the request and response +must pass through libsy's neutral types. Switchyard's in-memory preservation +replays same-protocol provider fields exactly. When the target protocol differs, +Switchyard translates only the common representable contract and rejects +configured loss. -```bash -cargo build -p nemo-relay-cli --features switchyard -``` +## Try the No-Service Example -Run the real-service compatibility test from the NeMo Relay repository root: +From a Relay source checkout: ```bash examples/switchyard/run-real-e2e.sh ``` -The script prints the verified Switchyard revision and ends with the following -line when routing succeeds: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -The test starts these local processes: - -| Process | Port | Readiness Check or Purpose | -| --- | --- | --- | -| Switchyard Decision API | `4000` | Serves `/health`, routing decisions, and ATOF ingestion. | -| NeMo Relay | `4041` | Serves `/healthz` and the OpenAI-compatible gateway. | -| Deterministic provider | `4101` | Records the model selected for each provider request. | - -The first request has no accumulated ATOF history and routes to -`provider/weak`. The script then sends session and tool events to Relay. Relay -exports those events to Switchyard's ATOF accumulator, so the next buffered -request and the final streaming request route to `provider/strong`. The test -also validates the streamed response before reporting success. - -## Configuration Walkthrough +The example starts Relay and one fake provider, exercises deterministic routing +between two semantic targets, and validates buffered and streaming fidelity. -The compatibility test uses -[`real-e2e-plugins.toml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-plugins.toml) -and -[`real-e2e-profiles.yaml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-profiles.yaml). -The following settings establish the routing boundary: - -| Setting | Purpose | -| --- | --- | -| `decision_api_url` | Points Relay to the separately running Switchyard Decision API. | -| `decision_profile_id` | Selects the Switchyard routing profile. The profile maps its semantic targets to backend IDs. | -| `targets.` | Binds each returned backend ID to a Relay-owned model, protocol, endpoint, base URL, and credentials. | -| `mode = "enforce"` | Applies a valid Switchyard decision and dispatches to the selected target. | -| `mode = "observe_only"` | Records the hypothetical decision but dispatches once to the trusted same-protocol default. | -| `default_targets` | Defines trusted same-protocol fallbacks for unavailable or invalid decisions and provider failures. | -| `context_mode = "atof_required"` | Requires stable identity and accumulated ATOF history for the selected profile. | -| `atof_endpoint_name` | Selects exactly one named Relay ATOF HTTP stream sink that sends events to Switchyard. | - -For history-based routing, a local ATOF JSONL file is not sufficient. -Switchyard must receive events through the named HTTP sink at -`/v1/atof/events` so its accumulator can provide the history used by the -routing profile. Relay rejects a missing, duplicate, disabled, or invalid -named sink during startup validation. - -For the complete option reference, refer to -[Switchyard Configuration](./configuration.mdx). - -## Troubleshooting - -Use the following table to diagnose common compatibility-test failures: - -| Symptom | Cause and Resolution | -| --- | --- | -| `Switchyard worktree not found: ...` | Place the Switchyard checkout next to `NeMo-Relay` as `Switchyard-topic-nemo-relay-integration`, or set `SWITCHYARD_ROOT` to its absolute path. | -| `Switchyard checkout mismatch: expected ..., found ...` | Check out commit `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8`. Override `SWITCHYARD_EXPECTED_COMMIT` only when deliberately testing a different contract. | -| A build uses Rust `1.96.0` or reports an unsupported toolchain | Run `rustc --version`, install Rust `1.96.1`, and allow the repository's `rust-toolchain.toml` to select it. | -| `timed out waiting for http://127.0.0.1:4000/health` | Review `switchyard.log`. Confirm the pinned checkout builds and port `4000` is free. | -| `timed out waiting for http://127.0.0.1:4041/healthz` or `process ... exited before http://127.0.0.1:4041/healthz became ready` | Review `relay.log`. Confirm the CLI was built with `--features switchyard`, the Switchyard health check passed, and port `4041` is free. | -| The provider cannot start or requests fail | Review `upstream.log` and confirm port `4101` is free. | -| `E2E logs preserved in ...` | The script preserves its temporary directory after a runtime failure and prints the exact path. Inspect `switchyard.log`, `relay.log`, `upstream.log`, and the captured request outputs there. Successful runs remove the directory. | - -## Capabilities - -The integration provides the following capabilities: - -- `enforce` and `observe_only` routing modes. -- OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages inbound - profiles. -- Buffered and streaming protocol translation through Switchyard's - `switchyard-translation` library. -- Exact Relay-owned backend bindings and per-protocol trusted fallbacks. -- Bounded retries before the first streaming item. -- Canonical routing marks and shared LLM optimization accounting. - -## Pages - -For more information, refer to the following pages: - -- [Switchyard Configuration](./configuration.mdx) - provides the complete option and deployment reference. Review its - [experimental limitations](./configuration.mdx#experimental-limitations) - before adopting the integration. -- [Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard) - contains the versioned configuration and validation scripts used by this - guide. +Continue with [Switchyard Configuration](./configuration.mdx) for the complete +version-2 schema. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx index 54bce55c0..d61fd937c 100644 --- a/docs/configure-plugins/switchyard/configuration.mdx +++ b/docs/configure-plugins/switchyard/configuration.mdx @@ -1,48 +1,17 @@ --- title: "Switchyard Configuration" sidebar-title: "Configuration" -description: "Configure the experimental Relay-native Switchyard Decision API plugin." +description: "Configure the library-only Relay-native Switchyard plugin." position: 2 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -> **Experimental NeMo Relay 0.6.0 integration:** This release calls a separately -> running Switchyard Decision API. +The Switchyard component runs libsy in process. It currently supports the +random router in enforce mode for OpenAI Chat, OpenAI Responses, and Anthropic +Messages managed LLM calls. -Start with the -[Switchyard 0.6.0 setup and validation guide](./about.mdx) -before using this page as the complete configuration reference. - -The Switchyard component connects Relay's CLI gateway to a separately running -[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API. Relay -owns provider credentials, backend endpoints, dispatch, retries, and trusted -fallbacks. The Switchyard service selects a backend. Relay validates the -decision, translates requests and responses in process through the -`switchyard-translation` library, and dispatches directly to the selected -Relay-owned target. - -## How the Configuration Fits Together - -Configure the integration in four core steps. Profiles that use ATOF history -add the fifth requirement: - -1. Set `decision_api_url` and authentication so Relay can call the Switchyard - Decision API. -2. Select the Switchyard routing profile with `decision_profile_id`. -3. Add a Relay-owned `targets` binding for every Switchyard `backend_id` that - the profile can return. Relay validates the returned backend ID, model, - protocol, and endpoint exactly before dispatch. -4. Choose a trusted same-protocol target for each inbound protocol under - `default_targets`. Relay uses these defaults for observe-only traffic and - fail-open dispatch. -5. Configure profiles that depend on ATOF history, including StageRouter, with - `context_mode = "atof_required"` and `atof_endpoint_name`, then configure the - matching observability stream sink. Relay fails startup validation if the - named sink is missing or invalid. - -The following is a complete component configuration. The referenced -environment variables must be set before Relay starts. +## Complete Example ```toml version = 1 @@ -52,307 +21,126 @@ kind = "switchyard" enabled = true [components.config] -version = 1 -mode = "enforce" +version = 2 priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] +enabled_inbound_profiles = [ + "openai_chat", + "openai_responses", + "anthropic_messages", +] -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] -openai_chat = "openai-default" +openai_chat = "chat-default" openai_responses = "responses-default" anthropic_messages = "anthropic-default" -[components.config.targets.openai-default] -model = "provider/efficient-chat" +[components.config.targets.fast] +model = "provider/fast" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://provider.example.com" +weight = 1 + +[components.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" -[components.config.targets.openai-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" +[components.config.targets.chat-default] +model = "provider/chat-fallback" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 0 [components.config.targets.responses-default] -model = "provider/efficient-responses" +model = "provider/responses-fallback" protocol = "openai_responses" endpoint = "/v1/responses" base_url = "https://provider.example.com" - -[components.config.targets.responses-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" +weight = 0 [components.config.targets.anthropic-default] -model = "provider/efficient-messages" +model = "provider/anthropic-fallback" protocol = "anthropic_messages" endpoint = "/v1/messages" base_url = "https://provider.example.com" - -[components.config.targets.anthropic-default.header_env] -x-api-key = "ANTHROPIC_PROVIDER_API_KEY" - -[components.config.targets.capable-chat] -model = "provider/capable-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.capable-chat.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 2 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +weight = 0 ``` -## Decision API and Rollout Options - -The following table describes the Decision API and rollout settings: +## Component Fields -| Option | Default | Description and constraints | +| Field | Default | Description | | --- | --- | --- | -| `version` | `1` | Component configuration version. | -| `mode` | `"enforce"` | `enforce` applies valid decisions. `observe_only` records the hypothetical decision but dispatches the trusted same-protocol default once. | -| `priority` | `0` | Component execution priority. | -| `decision_api_url` | Required | HTTP or HTTPS URL for the Switchyard Decision API. | -| `decision_profile_id` | Required | Non-empty Switchyard profile identifier sent with each routing request. | -| `decision_timeout_millis` | `25` | Decision API timeout in milliseconds; must be greater than zero. | -| `decision_headers` | Empty | Optional non-sensitive static Decision API headers. | -| `decision_header_env` | Empty | Map of header names to environment-variable names for sensitive Decision API headers. Referenced values must be present and non-blank. | - -Do not define the same header, including case variants, in both a static header -map and its environment map. Keep bearer tokens, API keys, cookies, and other -secrets in environment variables rather than tracked configuration. - -## Request Materialization - -Every Decision API request contains routing identity, inbound protocol, a -request summary, and the attempt number. `request_materialization` controls how -much additional request content is included. +| `version` | `2` | Library-only schema version. Version 1 is rejected because it used the removed Decision API service contract. | +| `priority` | `0` | Relay LLM execution-intercept priority. | +| `max_retries` | `3` | Provider retries after the initial libsy run. Maximum: `10`. | +| `enabled_inbound_profiles` | All three protocols | Managed Relay profiles intercepted by this component. | +| `algorithm` | Random without a seed | libsy algorithm configuration. Only `kind = "random"` is supported. | +| `targets` | Required | Semantic target names mapped to Relay-owned provider bindings. | +| `default_targets` | Required per enabled profile | Same-protocol targets used for trusted fallback. | -| Mode | Material sent to Switchyard | -| --- | --- | -| `none` | No `current_request`; use identity, protocol, summary, and attempt metadata only. | -| `summary_only` | No `current_request`; explicitly selects summary-based routing. | -| `latest_user_prompt` | A provider-valid body containing the latest user material plus an explicit prompt field. The request fails open if no user message is available. | -| `recent_message_window` | The most recent normalized messages, bounded by `recent_message_count`, re-encoded for the inbound protocol. | -| `annotated_request` | The final intercepted provider body plus Relay's normalized request representation. | -| `full_body` | The final intercepted provider request body. | +### Random Algorithm -Choose the least materialization needed by the selected profile. In particular, -`none` and `summary_only` intentionally share the same current wire behavior but -express different configuration intent. +`seed` is optional. Configure it for repeatable QA sequences. Each target's +nonnegative `weight` participates in libsy's weighted selection; `0` excludes a +target from normal routing while retaining it as a trusted fallback. -## Context, Retries, and Inbound Protocols +### Target Bindings -The following table describes context, retry, and inbound protocol settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `context_mode` | Required | `payload_only` or `atof_required`. See below. | -| `max_retries` | `3` | Provider retries after the initial attempt; maximum `10`. Each retry obtains a new Switchyard decision. | -| `recent_message_count` | `8` | Message limit for `recent_message_window`; must be greater than zero. | -| `enabled_inbound_profiles` | All three supported protocols | Non-empty list containing `openai_chat`, `openai_responses`, and/or `anthropic_messages`. | -| `atof_endpoint_name` | Required for `atof_required` | Name of exactly one stream sink under the observability component's `atof.sinks` list. | - -`payload_only` supports request-aware routing without an ATOF dependency. -Relay can create request-scoped synthetic identity when stable identity is not -available. - -`atof_required` is for profiles that classify from accumulated ATOF context. It -requires stable session and request identity plus an enabled, authenticated -Relay HTTP ATOF stream sink whose `name` matches `atof_endpoint_name`. The sink -specifies the Switchyard ingestion URL, must use `transport = "http_post"`, and -must set `field_name_policy = "preserve"`. Relay rejects missing or duplicate -named sinks during startup validation. A local ATOF file sink -does not satisfy this delivery requirement by itself. - -## Target Bindings and Trusted Defaults - -The following table describes target bindings and trusted defaults: - -| Option | Required fields | Purpose | +| Field | Required | Description | | --- | --- | --- | -| `targets.` | `model`, `protocol`, `endpoint`, `base_url` | Binds an exact Switchyard backend ID to a Relay-owned provider destination. | -| `targets..headers` | None | Optional non-sensitive static provider headers. | -| `targets..header_env` | None | Optional provider header names mapped to environment-variable names. Referenced values must be present and non-blank. | -| `default_targets.openai_chat` | Target using `openai_chat` | Trusted fallback for OpenAI Chat Completions requests. | -| `default_targets.openai_responses` | Target using `openai_responses` | Trusted fallback for OpenAI Responses requests. | -| `default_targets.anthropic_messages` | Target using `anthropic_messages` | Trusted fallback for Anthropic Messages requests. | - -Do not define the same target header, including case variants, in both -`headers` and `header_env`. - -Protocol and endpoint pairs are fixed: - -| Protocol | Endpoint | -| --- | --- | -| `openai_chat` | `/v1/chat/completions` | -| `openai_responses` | `/v1/responses` | -| `anthropic_messages` | `/v1/messages` | - -Each protocol in `enabled_inbound_profiles` requires a corresponding -`default_targets` entry. Defaults for disabled protocols can be omitted. Each -configured default must refer to a target with the matching protocol. Duplicate -exact bindings for model, protocol, endpoint, and base URL are rejected. - -## Recommended Recipes - -### Observe-Only Rollout - -Use observe-only mode to verify decisions and routing marks without sending -traffic to the selected target: - -```toml -[components.config] -mode = "observe_only" -context_mode = "payload_only" -request_materialization = "summary_only" -``` - -Relay calls the Decision API and records the hypothetical selection, but sends -the request to the inbound protocol's trusted default once. Routing retries are -not applied. - -### Payload-Only Request-Aware Routing - -Use recent request content for routing without operating the ATOF accumulator: - -```toml -[components.config] -mode = "enforce" -context_mode = "payload_only" -request_materialization = "recent_message_window" -recent_message_count = 8 -``` - -### ATOF-Backed StageRouter Routing - -Use ATOF context when StageRouter should classify from accumulated session -signals: - -```toml -[components.config] -mode = "enforce" -decision_profile_id = "stage-router" -context_mode = "atof_required" -request_materialization = "summary_only" -atof_endpoint_name = "switchyard" -``` - -Configure the matching observability component with a named stream sink: - -```toml -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 2 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" -``` - -## Runtime and Fail-Open Behavior - -The integration handles routing and failures as follows: - -- Relay validates the decision's backend ID, model, protocol, and endpoint - against its target binding before dispatch. Unknown backends, malformed - decisions, and target drift fail open. -- Decision API errors are not recursively retried. Relay immediately dispatches - the trusted same-protocol default. -- Retryable provider connection, timeout, status, context-window, and - model-unavailable failures can obtain a new decision, bounded by - `max_retries`. -- Non-retryable provider failures use the trusted fallback immediately. After - retry exhaustion, Relay dispatches that fallback once. -- A streaming request can retry only before receiving its first upstream item. - After the stream commits, Relay propagates later failures without another - dispatch. -- Relay forwards provider-specific extensions unchanged when every configured - target uses the inbound protocol. If any target uses another protocol, - nonportable extensions fail open to the trusted same-protocol default before - the Decision API call. Request translation failures before provider dispatch - and buffered response translation failures also fail open. After a streaming - response emits its first item, later provider or translation failures - propagate without another dispatch. -- Routing-mark delivery is best-effort and does not change provider results. - -## Experimental Limitations - -Consider the following experimental limitations before adopting the plugin: - -- The plugin is published as `nemo-relay-switchyard` but is excluded from - default Relay CLI builds. Build the CLI with - `cargo build -p nemo-relay-cli --features switchyard`. -- The routing boundary is currently service-based. Relay calls Switchyard's - HTTP Decision API but does not start or supervise the service. Relay derives - `/health` from `decision_api_url` and fails plugin activation unless the - service returns `{"status":"ok"}`. Decision API failures after activation fail - open to the trusted same-protocol default. -- ATOF-backed profiles additionally depend on Switchyard's HTTP ingestion and - accumulator runtime, stable request and session identity, and one named, - authenticated Relay ATOF HTTP stream sink. A local file sink does not populate - the Switchyard accumulator. -- Provider-protocol translation runs in Relay's process through the - `switchyard-translation` Rust library. Routing decisions and ATOF accumulation - remain out of process. -- The Rust Switchyard component is currently registered only by Relay's CLI - gateway. Hermes native plugins and the LangChain, LangGraph, and Deep Agents - integrations do not load it in process. They receive Switchyard routing only - when their provider traffic is explicitly sent through the Relay gateway. -- The Switchyard examples are manual compatibility and trajectory workflows, - not production deployment orchestration. -- Compatibility is pinned to Switchyard commit - [`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8) - on `topic/nemo-relay-integration`. Testing another revision is deliberate and - can expose contract drift. -- The Decision API and service/library boundary are experimental. -- Nonportable provider extensions in configurations that can translate across - protocols, malformed decisions, service failures, pre-dispatch translation - failures, buffered-response translation failures, and target drift fail open - to Relay's trusted same-protocol default. Failures after a streaming response - commits propagate without another dispatch. -- Secrets must remain environment-referenced. The examples generate ephemeral - local credentials and are not credential-management guidance. - -Run `nemo-relay doctor` to validate component configuration before startup. For -the pinned local service and manual compatibility workflow, refer to the -[Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard). +| `model` | Yes | Physical provider model. | +| `protocol` | Yes | `openai_chat`, `openai_responses`, or `anthropic_messages`. | +| `endpoint` | Yes | Must match the protocol's standard endpoint. | +| `base_url` | Yes | HTTP or HTTPS provider base URL. | +| `weight` | No | Relative random-routing weight; defaults to `1`. | +| `headers` | No | Static non-sensitive provider headers. | +| `header_env` | No | Header names mapped to environment variables, intended for credentials. | + +Caller credentials are removed before an explicit target dispatch and replaced +with the target's configured credentials from `header_env`. Relay identity +headers are mapped only into libsy `Context` and `Metadata`; the plugin does not +expose them to the algorithm as provider HTTP headers. + +## Translation Rules + +`switchyard-translation` is the component's only wire-format translation +engine. The component does not invoke NeMo Relay codecs for provider conversion. + +- Inbound provider JSON is decoded into Switchyard's neutral request before + `run_stream`. +- Every `CallLlm` request is encoded for the selected target protocol. +- Provider responses and stream events are decoded before + `CallLlmRequest::respond`. +- `ReturnToAgent` is encoded into the caller's original protocol. +- Same-protocol request bodies, buffered responses, and stream events preserve + provider-specific JSON through Switchyard's in-memory preservation contract. +- Cross-protocol routes reject lossy conversions rather than silently removing + a required feature. + +## Retry and Fallback + +Every retry starts a new libsy run and obtains a fresh random decision. Relay +returns the actual provider error to libsy before classifying the original typed +Relay error for retry. + +Streaming retries are allowed only before the first provider event. Once the +first event commits the stream, later errors are delivered to the caller and do +not trigger another provider call. + +When attempts are exhausted or a non-retryable routing or translation error +occurs, Relay dispatches the configured same-protocol fallback exactly once. + +## Current Limits + +- No decision-only or observe-only mode. +- No stage router or ensemble-specific concurrent dispatch. +- No Switchyard-owned provider client, service, or ATOF ingestion. +- No synthesized decision IDs, confidence, baselines, or optimization + contributions. +- Provider features that cannot be represented losslessly across protocols are + rejected. diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md index 87a2cff68..eec70cdd0 100644 --- a/examples/switchyard/README.md +++ b/examples/switchyard/README.md @@ -3,112 +3,34 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Switchyard Integration Examples +# Switchyard libsy example -These examples exercise the experimental NeMo Relay 0.6.0 integration with a separately running -Switchyard Decision API service and the in-process Switchyard translation library. They are -manual, local validation workflows rather than production startup orchestration. +This example runs Switchyard routing inside NeMo Relay. It starts only: -For the canonical architecture, setup, configuration, validation, and troubleshooting workflow, -refer to the -[Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about). +1. a deterministic local provider; +2. a Relay CLI process built with the `switchyard` feature. -## Required Switchyard Revision +There is no `switchyard-server`, Decision API, health check, or Switchyard ATOF +ingestion service. -The NeMo Relay 0.6.0 scripts require the following public topic branch and commit: - -```text -https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration -8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Clone the Switchyard repository next to the Relay checkout, then pin the -required commit: - -```bash -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - ../Switchyard-topic-nemo-relay-integration -git -C ../Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Every real-service script verifies this commit before launching `switchyard-server`. To test a -deliberately different checkout, set both variables explicitly: - -```bash -SWITCHYARD_ROOT=/path/to/Switchyard \ -SWITCHYARD_EXPECTED_COMMIT= \ - examples/switchyard/run-real-e2e.sh -``` - -## Examples - -Run these commands from the root of the NeMo Relay checkout. - -The Relay CLI's Switchyard support is compile-time optional. The scripts enable the `switchyard` -feature automatically; custom builds must pass `--features switchyard`. - -### Manual Switchyard Compatibility Smoke Test - -`run-real-e2e.sh` is a manual compatibility smoke test. It starts the pinned Switchyard server, -Relay, and a fake provider, then verifies cold and warm StageRouter decisions, buffered routing, -SSE routing, and the selected model sequence. It is intended to catch cross-repository service -contract regressions; the CI-safe Relay process regression test covers the faster local behavior -checks. The script requires Rust tooling, Python, and `curl`; its temporary logs are removed after -a successful run. +From the Relay repository root: ```bash examples/switchyard/run-real-e2e.sh ``` -A successful run ends with: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -### Hermes and Ollama Trajectory - -`run-hermes-ollama-smoke.sh` runs a fixed multi-query trajectory through Hermes, Relay, Ollama, -and Switchyard. It requires Docker, Hermes, and the configured local Ollama models. The script -produces ATOF, ATIF, and OTEL artifacts and can leave Phoenix running with -`SWITCHYARD_KEEP_PHOENIX=1`. - -```bash -examples/switchyard/run-hermes-ollama-smoke.sh -``` - -## Configuration Files - -The directory includes the following configuration and support files: - -- `plugins.toml`: minimal plugin configuration example. -- `real-e2e-plugins.toml` and `real-e2e-profiles.yaml`: deterministic fake-provider E2E. -- `hermes-ollama-plugins.toml` and `hermes-ollama-profiles.yaml`: local Ollama trajectory. -- `fake_upstream.py`: deterministic provider used by the service E2E. -- `otel-collector.yaml`: local OTEL artifact export configuration. - -## Runtime Model - -The scripts launch Switchyard as a separate local process on port `4000`. Relay sends routing -requests to `/v1/routing/decision` and, for ATOF-backed profiles, sends events to -`/v1/atof/events`. Relay owns provider credentials, target bindings, dispatch, retries, and -fallback behavior. Relay executes provider-protocol translation in process through Switchyard's -translation library; the Switchyard service owns ATOF accumulation and routing decisions. The -Switchyard component selects its HTTP ingestion destination by the observability stream sink name, -not by duplicating the sink URL. +The script sends buffered and streaming OpenAI Chat requests through a seeded, +weighted libsy random router. It verifies that both configured provider models +are selected, that Relay performs every physical request, and that a +provider-specific `system_fingerprint` survives the same-protocol stream round +trip. -The service is not started automatically by Relay outside these examples. A production deployment -must start a compatible Switchyard service before Relay activates the plugin and configure the -Relay plugin with its Decision API URL. Relay derives the service's root `/health` URL from that -configuration and refuses activation unless it reports `{"status":"ok"}`. +The files are: -## Artifacts and Troubleshooting +- `plugins.toml`: version-2 library-only Switchyard configuration; +- `fake_upstream.py`: deterministic OpenAI-compatible provider; +- `e2e-common.sh`: process and readiness helpers; +- `run-real-e2e.sh`: executable no-service smoke test. -Trajectory scripts write to `artifacts/` by default. Set `SWITCHYARD_TRAJECTORY_DIR` to choose a -shareable output directory. On failure, logs are preserved and include the verified Switchyard -revision. Do not place API keys or bearer tokens in configuration files; use environment variables -or an untracked secrets file. For service readiness failures, port conflicts, and the exact -temporary log locations used by `run-real-e2e.sh`, refer to the -[canonical Switchyard 0.6.0 guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about#troubleshooting). +`plugins.toml` contains no credentials. For real providers, use each target's +`header_env` map and keep secret values outside tracked configuration. diff --git a/examples/switchyard/e2e-common.sh b/examples/switchyard/e2e-common.sh index bf38627d6..f9687aeda 100755 --- a/examples/switchyard/e2e-common.sh +++ b/examples/switchyard/e2e-common.sh @@ -50,29 +50,6 @@ e2e_wait_for() { return 1 } -e2e_random_token() { - python3 -c 'import secrets; print(secrets.token_hex(24))' -} - -# The experimental Relay integration is tested against a specific public -# Switchyard topic-branch commit. Override this only when intentionally testing -# another revision. -e2e_verify_switchyard_checkout() { - local root="$1" - local expected="${2:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" - local actual - actual="$(git -C "$root" rev-parse HEAD 2>/dev/null)" || { - echo "Switchyard worktree is not a git checkout: $root" >&2 - return 1 - } - if [[ "$actual" != "$expected" ]]; then - echo "Switchyard checkout mismatch: expected $expected, found $actual" >&2 - echo "Set SWITCHYARD_EXPECTED_COMMIT only when intentionally testing another revision." >&2 - return 1 - fi - printf 'Switchyard revision: %s (%s)\n' "$actual" "$(git -C "$root" show -s --format=%s HEAD)" -} - e2e_tail_logs() { local directory="$1" local log diff --git a/examples/switchyard/fake_upstream.py b/examples/switchyard/fake_upstream.py index f27527d14..211cb0472 100644 --- a/examples/switchyard/fake_upstream.py +++ b/examples/switchyard/fake_upstream.py @@ -24,6 +24,7 @@ def do_POST(self) -> None: # noqa: N802 "id": "chatcmpl-fake", "object": "chat.completion.chunk", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [ { "index": 0, @@ -36,6 +37,7 @@ def do_POST(self) -> None: # noqa: N802 "id": "chatcmpl-fake", "object": "chat.completion.chunk", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ] @@ -52,6 +54,7 @@ def do_POST(self) -> None: # noqa: N802 "id": "chatcmpl-fake", "object": "chat.completion", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [ { "index": 0, diff --git a/examples/switchyard/hermes-ollama-plugins.toml b/examples/switchyard/hermes-ollama-plugins.toml deleted file mode 100644 index f562e2c72..000000000 --- a/examples/switchyard/hermes-ollama-plugins.toml +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "hermes-ollama-stage-router" -request_materialization = "full_body" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 120000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "ollama-weak" -openai_responses = "ollama-responses" -anthropic_messages = "ollama-anthropic" - -[components.config.targets.ollama-weak] -model = "llama3.2:latest" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-strong] -model = "qwen3.6:35b" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-responses] -model = "llama3.2:latest" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-anthropic] -model = "llama3.2:latest" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:11434" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" -output_directory = "." -filename = "trajectory.atof.jsonl" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.atif] -enabled = true -agent_name = "Hermes" -agent_version = "switchyard-stage-router-smoke" -model_name = "switchyard-stage-router" -output_directory = "." -filename_template = "trajectory-{session_id}.atif.json" - -[components.config.opentelemetry] -enabled = true -transport = "http_binary" -endpoint = "http://127.0.0.1:4318/v1/traces" -service_name = "nemo-relay-switchyard-e2e" -service_namespace = "switchyard-integration" -instrumentation_scope = "nemo-relay-switchyard" -timeout_millis = 5000 - -[components.config.opentelemetry.resource_attributes] -"deployment.environment.name" = "local-e2e" -"switchyard.profile.id" = "hermes-ollama-stage-router" diff --git a/examples/switchyard/hermes-ollama-profiles.yaml b/examples/switchyard/hermes-ollama-profiles.yaml deleted file mode 100644 index 751201851..000000000 --- a/examples/switchyard/hermes-ollama-profiles.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - ollama-weak: - model: llama3.2:latest - format: openai - base_url: http://127.0.0.1:11434/v1 - ollama-strong: - model: qwen3.6:35b - format: openai - base_url: http://127.0.0.1:11434/v1 - -profiles: - hermes-ollama-stage-router: - type: stage_router - capable: ollama-strong - efficient: ollama-weak - fallback_target_on_evict: ollama-strong - picker: efficient_first - # Force the classifier path for ordinary prompts so this smoke demonstrates - # request-complexity routing rather than relying on incidental tool signals. - confidence_threshold: 1.0 - # Each fixed trajectory transition is driven by the latest canonical tool - # result so older critical signals do not leak into the simple follow-up. - signal_recent_window: 1 - classifier: - # Keep classification small and deterministic; qwen3.6:35b remains the - # capable backend selected for the complex trajectory turn. - model: llama3.2:latest - api_key: ${OLLAMA_CLASSIFIER_API_KEY} - base_url: http://127.0.0.1:11434/v1 - timeout_secs: 120.0 - recent_turn_window: 1 - max_tokens: 128 - system_prompt: >- - Select exactly one model tier for the latest user request. Return only - JSON in the form {"tier":"efficient"} or {"tier":"capable"}. Choose efficient - for arithmetic, extraction, formatting, concise factual responses, or - other direct low-risk work. Choose capable for architecture, concurrency, - debugging, multi-step reasoning, subtle correctness constraints, or - requests requiring a defensible technical analysis. In the supplied - State line, severity at least 1.0 always means capable; tests_passed=true - with severity 0.0 means efficient. Classify the latest state even when earlier - conversation turns were more complex. diff --git a/examples/switchyard/otel-collector.yaml b/examples/switchyard/otel-collector.yaml deleted file mode 100644 index 330a87077..000000000 --- a/examples/switchyard/otel-collector.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - -exporters: - file/trajectory: - path: /artifacts/trajectory.otel.json - otlp/phoenix: - endpoint: phoenix:4317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [file/trajectory, otlp/phoenix] diff --git a/examples/switchyard/plugins.toml b/examples/switchyard/plugins.toml index d6a1a69ed..a17040169 100644 --- a/examples/switchyard/plugins.toml +++ b/examples/switchyard/plugins.toml @@ -8,68 +8,28 @@ kind = "switchyard" enabled = true [components.config] -mode = "enforce" +version = 2 priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "smart-stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] +max_retries = 2 +enabled_inbound_profiles = ["openai_chat"] -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" +openai_chat = "fast" -[components.config.targets.openai-default] -model = "weak-model" +[components.config.targets.fast] +model = "provider/fast" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "http://127.0.0.1:4101" +weight = 1 -[components.config.targets.responses-default] -model = "weak-model" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "weak-model" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.strong] -model = "strong-model" +[components.config.targets.quality] +model = "provider/quality" protocol = "openai_chat" endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4102" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 3000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +base_url = "http://127.0.0.1:4101" +weight = 1 diff --git a/examples/switchyard/real-e2e-plugins.toml b/examples/switchyard/real-e2e-plugins.toml deleted file mode 100644 index 541beb057..000000000 --- a/examples/switchyard/real-e2e-plugins.toml +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "remote-stage-router" -request_materialization = "summary_only" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 1000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "weak" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.strong] -model = "provider/strong" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.weak] -model = "provider/weak" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "provider/weak-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "provider/weak-anthropic" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-profiles.yaml b/examples/switchyard/real-e2e-profiles.yaml deleted file mode 100644 index 6127b46ef..000000000 --- a/examples/switchyard/real-e2e-profiles.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:4101/v1 - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:4101/v1 - -profiles: - remote-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: efficient_first - confidence_threshold: 0.7 diff --git a/examples/switchyard/run-hermes-ollama-smoke.sh b/examples/switchyard/run-hermes-ollama-smoke.sh deleted file mode 100755 index f390eb75e..000000000 --- a/examples/switchyard/run-hermes-ollama-smoke.sh +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$relay_root/examples/switchyard/e2e-common.sh" -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" -run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" -artifact_dir="${SWITCHYARD_TRAJECTORY_DIR:-$relay_root/artifacts/hermes-switchyard-$run_id}" -token="$(e2e_random_token)" -docker_network="switchyard-e2e-$run_id" -phoenix_container="switchyard-phoenix-$run_id" -collector_container="switchyard-otel-$run_id" -phoenix_port="${SWITCHYARD_PHOENIX_PORT:-6006}" -keep_phoenix="${SWITCHYARD_KEEP_PHOENIX:-0}" -collector_running=0 -phoenix_running=0 -network_created=0 - -mkdir -p "$artifact_dir/phoenix" -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" >"$artifact_dir/switchyard-revision.txt" - -cleanup() { - local status=$? - e2e_stop_processes - if [[ $collector_running -eq 1 ]]; then - docker rm -f "$collector_container" >/dev/null 2>&1 || true - fi - if [[ $phoenix_running -eq 1 && ( $status -ne 0 || "$keep_phoenix" != "1" ) ]]; then - docker rm -f "$phoenix_container" >/dev/null 2>&1 || true - phoenix_running=0 - fi - if [[ $network_created -eq 1 && $phoenix_running -eq 0 ]]; then - docker network rm "$docker_network" >/dev/null 2>&1 || true - fi - if [[ $status -ne 0 ]]; then - echo "Hermes/StageRouter smoke failed; artifacts preserved in $artifact_dir" >&2 - e2e_tail_logs "$artifact_dir" - fi -} -trap cleanup EXIT - -for dependency in cargo curl docker hermes jq python3 tar; do - command -v "$dependency" >/dev/null || { - echo "missing required command: $dependency" >&2 - exit 1 - } -done - -docker info >/dev/null -for model in llama3.2:latest qwen3.6:35b; do - curl --fail --silent http://127.0.0.1:11434/api/tags \ - | jq -e --arg model "$model" '.models[] | select(.name == $model)' >/dev/null || { - echo "required Ollama model is not installed: $model" >&2 - exit 1 - } -done - -docker network create "$docker_network" >/dev/null -network_created=1 -docker run --detach --rm \ - --name "$phoenix_container" \ - --network "$docker_network" \ - --network-alias phoenix \ - --publish "127.0.0.1:$phoenix_port:6006" \ - --env PHOENIX_WORKING_DIR=/mnt/data \ - --volume "$artifact_dir/phoenix:/mnt/data" \ - arizephoenix/phoenix:13.22 >"$artifact_dir/phoenix.container-id" -phoenix_running=1 -e2e_wait_for "http://127.0.0.1:$phoenix_port/" 240 0.5 - -docker run --detach --rm \ - --name "$collector_container" \ - --network "$docker_network" \ - --publish 127.0.0.1:4318:4318 \ - --volume "$relay_root/examples/switchyard/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \ - --volume "$artifact_dir:/artifacts" \ - otel/opentelemetry-collector-contrib:0.135.0 \ - --config=/etc/otelcol-contrib/config.yaml >"$artifact_dir/collector.container-id" -collector_running=1 - -( - cd "$switchyard_root" - OLLAMA_CLASSIFIER_API_KEY="$token" \ - SWITCHYARD_ATOF_BEARER_TOKEN="$token" \ - cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/hermes-ollama-profiles.yaml" --port 4000 -) >"$artifact_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" -e2e_wait_for http://127.0.0.1:4000/health 240 0.5 - -run_query() { - local sequence="$1" - local label="$2" - local query="$3" - local resume_id="${4:-}" - local -a resume_args=() - local before_lines=0 - local after_lines - local atif_path - if [[ -n "$resume_id" ]]; then - resume_args=(--resume "$resume_id") - fi - if [[ -f "$artifact_dir/trajectory.atof.jsonl" ]]; then - before_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - fi - ( - cd "$artifact_dir" - HERMES_HOME="$artifact_dir/hermes" \ - OPENAI_API_KEY=ollama \ - SWITCHYARD_AUTHORIZATION="Bearer $token" \ - cargo run --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli \ - --features switchyard -- \ - run --agent hermes \ - --plugin-config-path "$relay_root/examples/switchyard/hermes-ollama-plugins.toml" \ - -- chat --provider custom --model llama3.2:latest \ - --query "$query" ${resume_args[@]+"${resume_args[@]}"} \ - --toolsets terminal --quiet --max-turns 2 --ignore-rules - ) >"$artifact_dir/query-$sequence-$label.log" 2>&1 - after_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - printf '%s\t%s\t%s\t%s\n' "$sequence" "$label" "$before_lines" "$after_lines" \ - >> "$artifact_dir/query-event-ranges.tsv" - # Each query is a separate Relay process. Replay its persisted ATOF segment - # as a completion barrier before the next process starts; ingestion is - # idempotent, so events already delivered by the best-effort live exporter - # are reported as duplicates rather than applied twice. - sed -n "$((before_lines + 1)),${after_lines}p" "$artifact_dir/trajectory.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$artifact_dir/trajectory-$sequence-$label.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" - jq -e '(.batch.ingested_events + .batch.duplicate_events) >= 1' \ - "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" >/dev/null - atif_path="$(find "$artifact_dir" -maxdepth 1 -name 'trajectory-*.atif.json' -print \ - | grep -E '/trajectory-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.atif\.json$' \ - | head -1 || true)" - if [[ -z "$atif_path" ]]; then - echo "ATIF exporter did not produce a trajectory for query $sequence" >&2 - exit 1 - fi - mv "$atif_path" "$artifact_dir/trajectory-$sequence-$label.atif.json" -} - -emit_stage_router_signal() { - local sequence="$1" - local label="$2" - local output="$3" - local path="$artifact_dir/trajectory-signal-$sequence-$label.atof.jsonl" - python3 - "$path" "$session_id" "$label" "$output" <<'PY' -import datetime -import json -import pathlib -import sys -import uuid - -path = pathlib.Path(sys.argv[1]) -session_id, label, output = sys.argv[2:] -event_uuid = str(uuid.uuid4()) -base = { - "atof_version": "0.1", - "kind": "scope", - "uuid": event_uuid, - "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "name": "trajectory_fixture", - "category": "tool", - "category_profile": {"tool_call_id": f"fixture-{label}"}, - "metadata": { - "session_id": session_id, - "trajectory_fixture": True, - "trajectory_fixture_label": label, - }, -} -events = [ - {**base, "scope_category": "start", "data": {"label": label}}, - {**base, "scope_category": "end", "data": {"output": output}}, -] -path.write_text("".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events)) -PY - cat "$path" >> "$artifact_dir/trajectory.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$path" > "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" - jq -e '.batch.ingested_events == 2' \ - "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" >/dev/null -} - -simple_query='Return exactly the integer result of 17 + 25, with no explanation.' -complex_query='Do not call tools. Act as a principal concurrency engineer. Analyze a bounded lock-free MPMC queue that uses compare-and-swap on head and tail but no generation counters. Give a concrete ABA failure interleaving, identify the required C++ memory order on each publication and consumption edge, and propose the smallest defensible correction. Be precise about the linearization points.' -followup_query='Ignore the earlier technical topic. Reply with exactly SIMPLE_DONE and nothing else.' - -run_query 01 simple "$simple_query" -session_id="$(jq -r 'select(.name == "switchyard.routing.requested") | .data.session_id' "$artifact_dir/trajectory.atof.jsonl" | head -1)" -if [[ -z "$session_id" || "$session_id" == "null" ]]; then - echo "could not recover the Hermes session ID from the first routing mark" >&2 - exit 1 -fi -emit_stage_router_signal 02 complex \ - 'CUDA out of memory while analyzing the concurrent queue; critical failure requires careful recovery and a capable model.' -run_query 02 complex "$complex_query" "$session_id" -emit_stage_router_signal 03 simple-followup \ - 'All tests passed. The next request is a direct low-risk formatting response: return exactly SIMPLE_DONE.' -run_query 03 simple-followup "$followup_query" "$session_id" - -# Give the asynchronous OTLP exporter a short flush window, then stop the -# collector cleanly so its file exporter closes the shareable OTLP JSON file. -sleep 2 -docker stop --time 10 "$collector_container" >/dev/null -collector_running=0 - -python3 - "$artifact_dir" "$session_id" "$phoenix_port" "$simple_query" "$complex_query" "$followup_query" <<'PY' -import json -import pathlib -import sys - -root = pathlib.Path(sys.argv[1]) -session_id = sys.argv[2] -phoenix_port = sys.argv[3] -queries = sys.argv[4:] -atof_path = root / "trajectory.atof.jsonl" -events = [json.loads(line) for line in atof_path.read_text().splitlines() if line.strip()] -marks = [event for event in events if event.get("name", "").startswith("switchyard.routing.")] -decisions = [event for event in marks if event.get("name") == "switchyard.routing.decision"] - -expected_models = ["llama3.2:latest", "qwen3.6:35b", "llama3.2:latest"] -event_ranges = [] -for line in (root / "query-event-ranges.tsv").read_text().splitlines(): - sequence, label, start, end = line.split("\t") - event_ranges.append((sequence, label, int(start), int(end))) -representative_decisions = [] -for sequence, label, start, end in event_ranges: - segment = events[start:end] - decision = next( - (event for event in segment if event.get("name") == "switchyard.routing.decision"), - None, - ) - if decision is None: - raise SystemExit(f"query {sequence} ({label}) produced no successful routing decision") - representative_decisions.append(decision) -actual_models = [event.get("data", {}).get("selected_model") for event in representative_decisions] -if actual_models != expected_models: - raise SystemExit(f"unexpected StageRouter route sequence: {actual_models}; expected {expected_models}") - -required_mark_names = {"switchyard.routing.requested", "switchyard.routing.decision"} -if not required_mark_names.issubset({event.get("name") for event in marks}): - raise SystemExit("routing requested/decision marks were not both emitted") -for event in marks: - name = event.get("name") - if event.get("category") != "custom": - raise SystemExit(f"{name} did not use category=custom") - if event.get("category_profile", {}).get("subtype") != name: - raise SystemExit(f"{name} category_profile.subtype was not canonical") - schema = event.get("data_schema", {}) - if schema != {"name": "switchyard.routing_mark", "version": "1"}: - raise SystemExit(f"{name} had unexpected data_schema: {schema}") - metadata = event.get("metadata", {}) - if metadata.get("session_id") != session_id: - raise SystemExit(f"{name} did not mirror session identity") -for event in decisions: - data = event["data"] - for key in ("decision_id", "router", "routing_attempt", "backend_id", "selected_tier", "selected_model", "latency_ms", "rollout_mode"): - if key not in data or data[key] is None: - raise SystemExit(f"decision mark missing {key}: {data}") - -atif_paths = sorted(root.glob("trajectory-*.atif.json")) -if len(atif_paths) != 3: - raise SystemExit(f"expected three ATIF trajectories, found {len(atif_paths)}") -for path in atif_paths: - payload = json.loads(path.read_text()) - if not payload.get("steps"): - raise SystemExit(f"ATIF trajectory has no steps: {path.name}") - -otel_path = root / "trajectory.otel.json" -if not otel_path.exists() or not otel_path.read_text().strip(): - raise SystemExit("OTLP file exporter did not produce trajectory.otel.json") -otel_batches = [json.loads(line) for line in otel_path.read_text().splitlines() if line.strip()] - -summary = { - "session_id": session_id, - "queries": [{ - "sequence": index + 1, - "input": query, - "selected_model": actual_models[index], - "selected_tier": representative_decisions[index]["data"].get("selected_tier"), - "reason_code": representative_decisions[index]["data"].get("reason_code"), - "reason_summary": representative_decisions[index]["data"].get("reason_summary"), - } for index, query in enumerate(queries)], - "expected_route_sequence": expected_models, - "actual_route_sequence": actual_models, - "routing_basis": [ - "cold StageRouter efficient default", - "canonical ATOF critical-error tool result (capable override)", - "canonical ATOF clean-tests tool result (efficient classifier decision)", - ], - "atof": {"file": atof_path.name, "event_count": len(events), "routing_mark_count": len(marks)}, - "atif": {"files": [path.name for path in atif_paths], "trajectory_count": len(atif_paths)}, - "otel": {"file": otel_path.name, "export_batch_count": len(otel_batches)}, - "phoenix_url": f"http://127.0.0.1:{phoenix_port}", -} -(root / "trajectory-summary.json").write_text(json.dumps(summary, indent=2) + "\n") -readme = f"""# Hermes / Ollama / Switchyard StageRouter trajectory - -This bundle captures one fixed three-query Hermes session routed through NeMo -Relay and the Switchyard Decision API. The verified representative route is: - -1. `llama3.2:latest` (efficient) — cold StageRouter default -2. `qwen3.6:35b` (capable) — critical-signal StageRouter override -3. `llama3.2:latest` (efficient) — clean-state classifier decision - -Session ID: `{session_id}` - -## Important fixture note - -The `CUDA out of memory` text in `trajectory-signal-02-complex.atof.jsonl` is an -intentional, synthetic ATOF tool-result fixture. The machine did not run out of -memory. Fixture events carry `metadata.trajectory_fixture = true` and a fixture -label so they cannot be confused with organic Hermes events. - -The fixtures are necessary for this demonstration because the current -Switchyard StageRouter Decision API classifies from its accumulated ATOF snapshot, -not directly from `current_request.body`. The critical fixture exercises the -real capable override. The clean-tests fixture removes the prior critical signal -from the one-result window and exercises the efficient classifier path. - -## File map - -| Files | Contents | Test coverage | -| --- | --- | --- | -| `trajectory-summary.json` | Machine-readable queries, selected models, reasons, counts, and Phoenix URL | Confirms expected and actual representative routes match | -| `trajectory.atof.jsonl` | Complete Relay ATOF stream for all three queries and the labeled fixtures | Identity propagation, lifecycle events, routing marks, and accumulator input | -| `trajectory-01-simple.atof.jsonl` | ATOF emitted by the first Hermes invocation | Cold-start efficient default | -| `trajectory-02-complex.atof.jsonl` | ATOF emitted by the complex Hermes invocation | Dispatch through the selected capable backend | -| `trajectory-03-simple-followup.atof.jsonl` | ATOF emitted by the final Hermes invocation | Return to the efficient backend | -| `trajectory-signal-*.atof.jsonl` | Canonical, labeled tool start/end fixtures | Capable critical-error override and efficient clean-state classification | -| `*.atof-ingest.json` | Switchyard ingestion reports for query segments and fixtures | Successful or idempotent ATOF accumulation | -| `trajectory-01-simple.atif.json` | ATIF representation of query 1 | Efficient-model trajectory structure | -| `trajectory-02-complex.atif.json` | ATIF representation of query 2 | Capable-model trajectory structure | -| `trajectory-03-simple-followup.atif.json` | ATIF representation of query 3 | Efficient follow-up trajectory structure | -| `trajectory.otel.json` | OTLP JSON batches written by the OpenTelemetry Collector | Relay spans exported to the collector and forwarded to Phoenix | -| `query-*.log` | Hermes/Relay stdout and stderr for each invocation | Human-readable harness responses and execution diagnostics | -| `query-event-ranges.tsv` | Query label and ATOF line-count boundaries | Separates representative user-query decisions from extra Hermes calls | - -## Routing-mark assertions - -The smoke validates every `switchyard.routing.*` mark in the cumulative ATOF -stream. Each mark must have: - -- `category: "custom"` -- `category_profile.subtype` equal to the mark name -- `data_schema.name: "switchyard.routing_mark"` -- `data_schema.version: "1"` -- the expected session identity in metadata - -Decision marks must also include a decision ID, router, attempt, backend, tier, -model, latency, and rollout mode. This run produced {len(marks)} routing marks -across {len(events)} total ATOF events. - -## Phoenix and OTLP - -During the smoke, Relay sends OTLP/HTTP to a local OpenTelemetry Collector. The -collector writes `trajectory.otel.json` and forwards the same spans to Phoenix. -The run produced {len(otel_batches)} OTLP export batches. When the smoke is run -with `SWITCHYARD_KEEP_PHOENIX=1`, open the `phoenix_url` from -`trajectory-summary.json` before stopping the printed Phoenix container. - -## Reproduce - -Install both `llama3.2:latest` and `qwen3.6:35b` in Ollama, ensure the cumulative -Switchyard checkout is available, then run: - -```bash -SWITCHYARD_KEEP_PHOENIX=1 examples/switchyard/run-hermes-ollama-smoke.sh -``` - -The script validates the route, mark shape, ATIF contents, and OTLP output before -creating this bundle. -""" -(root / "TRAJECTORY_README.md").write_text(readme) -print(json.dumps(summary, indent=2)) -PY - -( - cd "$artifact_dir" - tar -czf trajectory-bundle.tar.gz \ - TRAJECTORY_README.md \ - trajectory-summary.json \ - trajectory.atof.jsonl \ - trajectory.otel.json \ - trajectory-*.atif.json \ - trajectory-*.atof.jsonl \ - trajectory-*.atof-ingest.json \ - query-*.log \ - query-event-ranges.tsv -) - -echo "Hermes/Ollama StageRouter trajectory passed: llama3.2 -> qwen3.6:35b -> llama3.2" -echo "Artifacts: $artifact_dir" -echo "Bundle: $artifact_dir/trajectory-bundle.tar.gz" -if [[ "$keep_phoenix" == "1" ]]; then - echo "Phoenix: http://127.0.0.1:$phoenix_port (container $phoenix_container left running)" -else - echo "Set SWITCHYARD_KEEP_PHOENIX=1 to leave Phoenix running after the smoke." -fi diff --git a/examples/switchyard/run-real-e2e.sh b/examples/switchyard/run-real-e2e.sh index 1f6810871..1d20fd99e 100755 --- a/examples/switchyard/run-real-e2e.sh +++ b/examples/switchyard/run-real-e2e.sh @@ -6,19 +6,8 @@ set -euo pipefail relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "$relay_root/examples/switchyard/e2e-common.sh" -relay_toolchain="" -if command -v rustup >/dev/null 2>&1; then - relay_toolchain="$(cd "$relay_root" && rustup show active-toolchain)" - relay_toolchain="${relay_toolchain%% *}" -fi -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" work_dir="$(mktemp -d)" upstream_log="$work_dir/upstream.jsonl" -token="$(e2e_random_token)" - -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" cleanup() { local status=$? @@ -32,58 +21,45 @@ cleanup() { } trap cleanup EXIT +for dependency in cargo curl python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done + python3 "$relay_root/examples/switchyard/fake_upstream.py" \ --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & e2e_add_pid "$!" ( - cd "$switchyard_root" - SWITCHYARD_ATOF_BEARER_TOKEN="$token" cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/real-e2e-profiles.yaml" --port 4000 -) >"$work_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" - -e2e_wait_for http://127.0.0.1:4000/health - -( - if [[ -n "$relay_toolchain" ]]; then - export RUSTUP_TOOLCHAIN="$relay_toolchain" - fi - cd "$work_dir" - SWITCHYARD_AUTHORIZATION="Bearer $token" cargo run \ - --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli --features switchyard -- \ - --plugin-config-path "$relay_root/examples/switchyard/real-e2e-plugins.toml" \ + cd "$relay_root" + cargo run -p nemo-relay-cli --features switchyard -- \ + --plugin-config-path "$relay_root/examples/switchyard/plugins.toml" \ --bind 127.0.0.1:4041 ) >"$work_dir/relay.log" 2>&1 & relay_pid="$!" e2e_add_pid "$relay_pid" -e2e_wait_for http://127.0.0.1:4041/healthz 120 0.25 "$relay_pid" +e2e_wait_for http://127.0.0.1:4041/healthz 240 0.25 "$relay_pid" -request() { - local request_id="$1" - local stream="$2" - curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ +for request in $(seq 1 8); do + curl --fail --silent http://127.0.0.1:4041/v1/chat/completions \ -H 'content-type: application/json' \ - -H 'x-nemo-relay-session-id: e2e-session' \ - -H "x-nemo-relay-request-id: $request_id" \ - --data-binary "{\"model\":\"client-model\",\"stream\":$stream,\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}]}" -} - -request cold-request false >"$work_dir/cold.json" - -for payload in \ - '{"hook_event_name":"on_session_start","session_id":"e2e-session"}' \ - '{"hook_event_name":"pre_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' \ - '{"hook_event_name":"post_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"tool_response":{"output":"CUDA out of memory"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' -do - curl --fail --silent http://127.0.0.1:4041/hooks/hermes \ - -H 'content-type: application/json' --data-binary "$payload" >/dev/null + -H 'x-nemo-relay-session-id: libsy-example' \ + -H "x-nemo-relay-request-id: buffered-$request" \ + --data-binary \ + '{"model":"client/model","stream":false,"messages":[{"role":"user","content":"hello"}]}' \ + >"$work_dir/buffered-$request.json" done -sleep 1 -request warm-request false >"$work_dir/warm.json" -request stream-request true >"$work_dir/stream.sse" +curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: libsy-example' \ + -H 'x-nemo-relay-request-id: stream-1' \ + --data-binary \ + '{"model":"client/model","stream":true,"messages":[{"role":"user","content":"stream hello"}]}' \ + >"$work_dir/stream.sse" python3 - "$upstream_log" "$work_dir/stream.sse" <<'PY' import json @@ -92,10 +68,12 @@ import sys records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] models = [record["body"]["model"] for record in records] -if models != ["provider/weak", "provider/strong", "provider/strong"]: - raise SystemExit(f"unexpected cold/warm/stream route sequence: {models}") +if set(models) != {"provider/fast", "provider/quality"}: + raise SystemExit(f"seeded libsy router did not exercise both targets: {models}") stream = pathlib.Path(sys.argv[2]).read_text() if "fake" not in stream or "[DONE]" not in stream: raise SystemExit(f"unexpected SSE output: {stream}") -print(f"real Switchyard E2E passed: {models}") +if '"system_fingerprint":"fp_switchyard_example"' not in stream: + raise SystemExit(f"provider stream extension was not preserved: {stream}") +print(f"in-process Switchyard libsy E2E passed: {models}") PY From 39e757abc29927d1ebe7de817af67d63a08ffbda Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 29 Jul 2026 19:48:19 -0600 Subject: [PATCH 2/5] feat(switchyard): add LLM classifier routing Signed-off-by: Bryan Bednarski --- crates/cli/src/plugins/editor_model.rs | 1 + crates/cli/tests/switchyard_process_e2e.rs | 210 ++++++- crates/switchyard/LIBSY_GAPS.md | 48 +- crates/switchyard/README.md | 28 +- crates/switchyard/src/component.rs | 130 +++- .../switchyard/tests/unit/component_tests.rs | 561 ++++++++++++++++++ docs/configure-plugins/switchyard/about.mdx | 14 +- .../switchyard/configuration.mdx | 60 +- examples/switchyard/README.md | 27 +- examples/switchyard/classifier-plugins.toml | 43 ++ examples/switchyard/fake_upstream.py | 18 +- examples/switchyard/run-classifier-e2e.sh | 87 +++ 12 files changed, 1178 insertions(+), 49 deletions(-) create mode 100644 examples/switchyard/classifier-plugins.toml create mode 100755 examples/switchyard/run-classifier-e2e.sh diff --git a/crates/cli/src/plugins/editor_model.rs b/crates/cli/src/plugins/editor_model.rs index 0680f6586..df0ecabab 100644 --- a/crates/cli/src/plugins/editor_model.rs +++ b/crates/cli/src/plugins/editor_model.rs @@ -1181,6 +1181,7 @@ pub(super) fn switchyard_configured(config: &SwitchyardConfig) -> bool { pub(super) fn switchyard_summary(state: &ComponentEditorState) -> String { let algorithm = match &state.config.algorithm { AlgorithmConfig::Random { .. } => "random", + AlgorithmConfig::LlmClassifier { .. } => "llm classifier", }; format!( "component {}, algorithm {}, targets {}", diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs index da78abed3..ba0411dfe 100644 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ b/crates/cli/tests/switchyard_process_e2e.rs @@ -15,6 +15,8 @@ use axum::routing::post; use axum::{Json, Router}; use serde_json::{Value, json}; +const CLASSIFIER_VERDICT: &str = r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.95,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#; + fn gateway_bin() -> &'static str { env!("CARGO_BIN_EXE_nemo-relay") } @@ -52,10 +54,15 @@ async fn provide( .unwrap(); } if stream { + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "streamed" + }; let first = json!({ "id": "chat-ci", "object": "chat.completion.chunk", "model": model, "system_fingerprint": "fp_process_e2e", - "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}, "finish_reason": null}] + "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}, "finish_reason": null}] }); let last = json!({ "id": "chat-ci", "object": "chat.completion.chunk", "model": model, @@ -69,9 +76,14 @@ async fn provide( .body(Body::from(body)) .unwrap(); } + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT.to_string() + } else { + format!("served by {model}") + }; let body = json!({ "id": "chat-ci", "object": "chat.completion", "model": model, - "choices": [{"index": 0, "message": {"role": "assistant", "content": format!("served by {model}")}, "finish_reason": "stop"}], + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} }); Response::builder() @@ -245,3 +257,197 @@ weight = 0 provider_task.abort(); } + +#[tokio::test(flavor = "multi_thread")] +async fn switchyard_plugin_runs_classifier_and_routed_calls_without_a_service() { + let provider_state = ProviderState::default(); + let provider_requests = Arc::clone(&provider_state.requests); + let (provider_url, provider_task) = start_server( + Router::new() + .route("/v1/chat/completions", post(provide)) + .with_state(provider_state), + ) + .await; + + let temp = tempfile::tempdir().unwrap(); + let atof_dir = temp.path().join("atof"); + std::fs::create_dir_all(&atof_dir).unwrap(); + let config_path = temp.path().join("plugins.toml"); + let config = format!( + r#"version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 2 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "{}" +filename = "events.jsonl" +mode = "overwrite" + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +version = 2 +max_retries = 0 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "weak" +strong_target = "strong" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.default_targets] +openai_chat = "strong" + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" + +[components.config.targets.weak] +model = "provider/weak" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" + +[components.config.targets.strong] +model = "provider/strong" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" +"#, + atof_dir.display() + ); + std::fs::write(&config_path, config).unwrap(); + + let address = unused_address(); + let gateway_url = format!("http://{address}"); + let stderr = std::fs::File::create(temp.path().join("gateway.log")).unwrap(); + let child = Command::new(gateway_bin()) + .arg("--plugin-config-path") + .arg(&config_path) + .arg("--bind") + .arg(address.to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::from(stderr)) + .spawn() + .unwrap(); + let mut gateway = ChildGuard(child); + let client = reqwest::Client::new(); + wait_for_gateway(&client, &gateway_url, &mut gateway.0).await; + + let buffered = client + .post(format!("{gateway_url}/v1/chat/completions")) + .header("x-nemo-relay-session-id", "classifier-buffered") + .json(&json!({ + "model": "client/model", + "messages": [{"role": "user", "content": "classify buffered"}] + })) + .send() + .await + .unwrap(); + assert!(buffered.status().is_success()); + let buffered: Value = buffered.json().await.unwrap(); + assert_eq!(buffered["model"], "provider/weak"); + + let streaming = client + .post(format!("{gateway_url}/v1/chat/completions")) + .header("x-nemo-relay-session-id", "classifier-streaming") + .json(&json!({ + "model": "client/model", + "stream": true, + "messages": [{"role": "user", "content": "classify streaming"}] + })) + .send() + .await + .unwrap(); + assert!(streaming.status().is_success()); + let streaming = streaming.text().await.unwrap(); + assert!(streaming.contains("streamed")); + assert!(streaming.contains("[DONE]")); + + let providers = provider_requests.lock().unwrap(); + let models = providers + .iter() + .map(|(_, body)| body["model"].as_str().unwrap()) + .collect::>(); + assert_eq!( + models, + vec![ + "provider/classifier", + "provider/weak", + "provider/classifier", + "provider/weak" + ] + ); + for (_, body) in providers + .iter() + .filter(|(_, body)| body["model"] == "provider/classifier") + { + assert!(body["response_format"].is_object()); + } + drop(providers); + + let events_path = atof_dir.join("events.jsonl"); + let mut events = String::new(); + for _ in 0..40 { + events = std::fs::read_to_string(&events_path).unwrap_or_default(); + if events.matches("switchyard.routing.call").count() >= 4 { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let events = events + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| { + event["name"] + .as_str() + .is_some_and(|name| name.starts_with("switchyard.routing.")) + }) + .collect::>(); + let calls = events + .iter() + .filter(|event| event["name"] == "switchyard.routing.call") + .collect::>(); + assert_eq!(calls.len(), 4, "routing events: {events:?}"); + assert!(calls.iter().all(|event| { + event["data"]["algorithm"] == "llm_task_classifier" + && event["data"]["semantic_target"] + .as_str() + .is_some_and(|target| matches!(target, "classifier" | "weak")) + })); + assert_eq!( + calls + .iter() + .filter(|event| event["data"]["is_routed_call"] == false) + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| { + event["name"] == "switchyard.routing.decision" + && event["data"]["semantic_target"] == "weak" + }) + .count(), + 2 + ); + + provider_task.abort(); +} diff --git a/crates/switchyard/LIBSY_GAPS.md b/crates/switchyard/LIBSY_GAPS.md index 860ca7bdf..04b0afdab 100644 --- a/crates/switchyard/LIBSY_GAPS.md +++ b/crates/switchyard/LIBSY_GAPS.md @@ -5,9 +5,10 @@ SPDX-License-Identifier: Apache-2.0 # libsy Contract Assessment -This assessment covers the Relay-native random-router integration driven -through `Algorithm::run_stream`. Relay performs provider calls and returns each -real response, stream, or error through `CallLlmRequest::respond`. +This assessment covers the Relay-native random-router and LLM-classifier +integrations driven through `Algorithm::run_stream`. Relay performs provider +calls and returns each real response, stream, or error through +`CallLlmRequest::respond`. The initial assessment uses Switchyard revision `c8ca731adec50c948739a71157ce283ed360ea8a`. Development revisions keep @@ -48,6 +49,11 @@ The pinned library supports the required random-router lifecycle: - buffered request and response translation with exact same-format preservation. +The same driver can serve the classifier's two-call lifecycle: a classifier +consultation followed by the selected weak or strong target. The classifier +prompt-role gap below must be resolved upstream before that integration is +ready for review. + ## LIBSY-GAP-003: Exact Raw Stream-Event Preservation **Description:** A provider stream event must survive the @@ -108,9 +114,43 @@ dedicated model-unavailable category. - **Relay test unblocked:** A contract test proving that model-unavailable reaches an algorithm as a distinct typed failure. +## LIBSY-GAP-005: Classifier Prompt Instruction Role + +**Description:** The LLM classifier's governing prompt must reach the provider +as a system-level instruction, not as ordinary user content. + +- **Priority:** P0 for classifier routing correctness. +- **Affected Relay behavior:** Relay dispatches the classifier `CallLlm` + exactly as libsy constructs it. It must not rewrite neutral message roles + around the Switchyard translation contract. +- **Pinned behavior:** `CapabilityJudge` inserts its prompt as + `Message { role: System }`, while the OpenAI Chat encoder maps system roles + found in the normal message list to `user`. The classifier therefore receives + its policy prompt at the wrong privilege level. +- **Reproducer:** Start `LlmTaskClassifier::run_stream`, inspect its first + `CallLlm`, encode that request as OpenAI Chat, and observe + `messages[0].role == "user"`. +- **Required behavior:** Build the classifier prompt in + `LlmRequest.instructions` with `Role::System`. Switchyard's protocol encoders + then map it to the provider's native system-instruction field. +- **Proposed API area:** `switchyard-libsy::algorithms::LlmTaskClassifier` + request construction; no Relay API or codec change. +- **Upstream work:** `feat/libsy-classifier-system-instruction`. +- **Switchyard issue/PR:** Pending focused upstream review. The Relay + development pin will include the resolving commit alongside the stream + preservation commit until compatible Switchyard 0.2 crates are published. +- **Acceptance:** The libsy classifier contract test observes one system + instruction and one user task message; Relay's focused contract probe + observes the provider-native system field for both supported classifier + protocols. +- **Relay test unblocked:** + `classifier_prompt_is_encoded_as_a_system_instruction` (ignored while Relay + is pinned to the affected Switchyard revision). + ## Deferred Capabilities Decision-only execution, observe-only routing, stage routing, ensemble-specific concurrent dispatch, structured decision identity, confidence, baselines, and optimization contributions are outside this integration. They are not mapped -around locally and do not block the random-router `run_stream` contract. +around locally and do not block the random-router or LLM-classifier +`run_stream` contract. diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md index 16b88e285..d64531de2 100644 --- a/crates/switchyard/README.md +++ b/crates/switchyard/README.md @@ -7,7 +7,8 @@ SPDX-License-Identifier: Apache-2.0 `nemo-relay-switchyard` is NeMo Relay's experimental in-process integration with [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard). -It runs libsy's random router inside Relay through `Algorithm::run_stream`. +It runs libsy's weighted random router or judge-backed LLM classifier inside +Relay through `Algorithm::run_stream`. Relay owns provider target bindings, credentials, transport, retries, fallback, and observability. libsy emits routing decisions and requests provider calls; @@ -67,6 +68,31 @@ Target-map keys are libsy semantic names. Relay configuration remains authoritative for the physical model, protocol, endpoint, URL, and credentials. Version-1 Decision API configuration is rejected with a migration error. +For classifier routing, configure one Relay target for the judge plus weak and +strong routed targets: + +```toml +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +``` + +The classifier consultation and the selected model call are both emitted by +`run_stream` and dispatched by Relay. Under the pinned lossless translation +policy, the classifier target must use OpenAI Chat or OpenAI Responses because +its request requires a structured response format. Weak and strong targets may +use any supported protocol. + Build and test the optional CLI integration with: ```bash diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs index 0c3eb60c8..c5672d62a 100644 --- a/crates/switchyard/src/component.rs +++ b/crates/switchyard/src/component.rs @@ -25,7 +25,7 @@ use nemo_relay::plugin::{ }; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json, json}; -use switchyard_libsy::algorithms::Random; +use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; use switchyard_libsy::{ Algorithm, CallLlmRequest, Context as LibsyContext, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request as LibsyRequest, Response as LibsyResponse, Step, @@ -153,6 +153,29 @@ pub enum AlgorithmConfig { #[serde(default)] seed: Option, }, + /// Judge-backed routing between efficient and capable targets. + LlmClassifier { + /// Semantic target used for the classifier consultation. + classifier_target: String, + /// Semantic target used for tasks the classifier considers efficient. + weak_target: String, + /// Semantic target used for tasks requiring the capable tier. + strong_target: String, + /// Lowest solve probability that selects the weak target. + base_threshold: f64, + /// Lowest classifier confidence that permits weak-target routing. + #[serde(default)] + min_confidence: f64, + /// Higher solve-probability floor for uncertain capability verdicts. + #[serde(default)] + capability_elevated_floor: Option, + /// Reuse the first routing decision for subsequent requests in a session. + #[serde(default)] + session_affinity: bool, + /// Key affinity from the first user message when session metadata is absent. + #[serde(default)] + message_hash_fallback: bool, + }, } impl Default for AlgorithmConfig { @@ -545,6 +568,7 @@ impl SwitchyardRuntime { self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); } Ok(Step::CallLlm(call)) => { + self.emit_call(call.get_decision(), attempt, mark_metadata.clone()); self.serve_buffered_call( *call, headers.clone(), @@ -584,6 +608,7 @@ impl SwitchyardRuntime { self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); } Ok(Step::CallLlm(call)) => { + self.emit_call(call.get_decision(), attempt, mark_metadata.clone()); self.serve_stream_call( *call, headers.clone(), @@ -613,6 +638,7 @@ impl SwitchyardRuntime { ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); let target_name = routed.decision.selected_model().to_string(); + let is_routed_call = routed.decision.is_routed_call(); let result = async { let metadata = routed.request.metadata.clone(); let (target_protocol, request) = @@ -620,7 +646,9 @@ impl SwitchyardRuntime { let response = match next(request).await { Ok(response) => response, Err(error) => { - remember_provider_error(&provider_error, &error); + if is_routed_call { + remember_provider_error(&provider_error, &error); + } return Err(flow_to_client_error(error, &target_name)); } }; @@ -646,6 +674,7 @@ impl SwitchyardRuntime { ) -> switchyard_libsy::Result<()> { let routed = call.get_routed().clone(); let target_name = routed.decision.selected_model().to_string(); + let is_routed_call = routed.decision.is_routed_call(); let result = async { let metadata = routed.request.metadata.clone(); let (target_protocol, request) = @@ -653,14 +682,18 @@ impl SwitchyardRuntime { let mut upstream = match next(request).await { Ok(upstream) => upstream, Err(error) => { - remember_provider_error(&provider_error, &error); + if is_routed_call { + remember_provider_error(&provider_error, &error); + } return Err(flow_to_client_error(error, &target_name)); } }; let first = match upstream.next().await { Some(Ok(first)) => first, Some(Err(error)) => { - remember_provider_error(&provider_error, &error); + if is_routed_call { + remember_provider_error(&provider_error, &error); + } return Err(flow_to_client_error(error, &target_name)); } None => { @@ -832,7 +865,7 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.requested", json!({ - "algorithm": "random", + "algorithm": self.algorithm.name(), "routing_attempt": attempt, "inbound_profile": inbound.label(), }), @@ -844,7 +877,22 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.decision", json!({ - "algorithm": "random", + "algorithm": self.algorithm.name(), + "routing_attempt": attempt, + "semantic_target": decision.selected_model(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + "reasoning": decision.reasoning(), + }), + metadata, + ); + } + + fn emit_call(&self, decision: &dyn Decision, attempt: u32, metadata: Json) { + emit_mark( + "switchyard.routing.call", + json!({ + "algorithm": self.algorithm.name(), "routing_attempt": attempt, "semantic_target": decision.selected_model(), "routing_tier": decision.routing_tier(), @@ -859,7 +907,7 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.retry", json!({ - "algorithm": "random", + "algorithm": self.algorithm.name(), "routing_attempt": attempt, "retry_reason": reason, }), @@ -871,7 +919,7 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.error", json!({ - "algorithm": "random", + "algorithm": self.algorithm.name(), "routing_attempt": attempt, "error_class": class, "error": error, @@ -884,7 +932,7 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.fallback", json!({ - "algorithm": "random", + "algorithm": self.algorithm.name(), "fallback_reason": reason, "fallback_route": self.config.default_targets.target(inbound), "inbound_profile": inbound.label(), @@ -916,25 +964,63 @@ impl RunFailure { } fn build_algorithm(config: &SwitchyardConfig) -> Result, String> { - let targets = config - .targets - .keys() - .map(|name| LlmTarget { - semantic_name: name.clone(), - llm_client: None, - }) - .collect::>(); - let weights = config - .targets - .values() - .map(|target| target.weight) - .collect::>(); + let target = |name: &str| { + config + .targets + .contains_key(name) + .then(|| LlmTarget { + semantic_name: name.to_string(), + llm_client: None, + }) + .ok_or_else(|| format!("algorithm target {name:?} is not configured")) + }; match &config.algorithm { AlgorithmConfig::Random { seed } => { + let targets = config + .targets + .keys() + .map(|name| target(name)) + .collect::, _>>()?; + let weights = config + .targets + .values() + .map(|target| target.weight) + .collect::>(); Random::new(LlmTargetSet::new(targets), Some(weights), *seed) .map(|algorithm| Arc::new(algorithm) as Arc) .map_err(|error| error.to_string()) } + AlgorithmConfig::LlmClassifier { + classifier_target, + weak_target, + strong_target, + base_threshold, + min_confidence, + capability_elevated_floor, + session_affinity, + message_hash_fallback, + } => { + let classifier = target(classifier_target)?; + if config.targets[classifier_target].protocol == WireProtocol::AnthropicMessages { + return Err(format!( + "classifier target {classifier_target:?} must use openai_chat or openai_responses because the pinned lossless translation policy cannot encode its structured response format for anthropic_messages" + )); + } + LlmTaskClassifier::new( + classifier, + target(weak_target)?, + target(strong_target)?, + TaskClassifierConfig { + base_threshold: *base_threshold, + min_confidence: *min_confidence, + capability_elevated_floor: *capability_elevated_floor, + session_affinity: *session_affinity, + message_hash_fallback: *message_hash_fallback, + }, + ) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } } } diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs index a252de07b..d17b6c4de 100644 --- a/crates/switchyard/tests/unit/component_tests.rs +++ b/crates/switchyard/tests/unit/component_tests.rs @@ -15,6 +15,8 @@ use switchyard_protocol::text_response; use super::*; +const CLASSIFIER_VERDICT: &str = r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.95,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#; + fn binding(protocol: WireProtocol, model: &str, weight: f64) -> TargetBinding { TargetBinding { model: model.into(), @@ -43,6 +45,88 @@ fn chat_config() -> SwitchyardConfig { } } +fn classifier_config(session_affinity: bool) -> SwitchyardConfig { + SwitchyardConfig { + algorithm: AlgorithmConfig::LlmClassifier { + classifier_target: "classifier".into(), + weak_target: "weak".into(), + strong_target: "strong".into(), + base_threshold: 0.5, + min_confidence: 0.5, + capability_elevated_floor: Some(0.8), + session_affinity, + message_hash_fallback: false, + }, + targets: BTreeMap::from([ + ( + "classifier".into(), + binding(WireProtocol::OpenaiChat, "provider/classifier", 1.0), + ), + ( + "weak".into(), + binding(WireProtocol::OpenaiChat, "provider/weak", 1.0), + ), + ( + "strong".into(), + binding(WireProtocol::OpenaiChat, "provider/strong", 1.0), + ), + ]), + default_targets: ProtocolDefaults { + openai_chat: "strong".into(), + ..ProtocolDefaults::default() + }, + enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiChat]), + ..SwitchyardConfig::default() + } +} + +async fn run_buffered_classifier(classifier_protocol: WireProtocol) -> (Json, Vec) { + let mut config = classifier_config(false); + config.targets.insert( + "classifier".into(), + binding( + classifier_protocol, + match classifier_protocol { + WireProtocol::OpenaiChat => "provider/classifier", + WireProtocol::OpenaiResponses => "provider/classifier-responses", + WireProtocol::AnthropicMessages => unreachable!(), + }, + 1.0, + ), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request.clone()); + if model.starts_with("provider/classifier") { + return Ok(match classifier_protocol { + WireProtocol::OpenaiChat => { + assert!(request.content["response_format"].is_object()); + chat_response(&model, CLASSIFIER_VERDICT) + } + WireProtocol::OpenaiResponses => { + assert!(request.content["text"]["format"].is_object()); + responses_response(&model, CLASSIFIER_VERDICT) + } + WireProtocol::AnthropicMessages => unreachable!(), + }); + } + Ok(chat_response(&model, "efficient answer")) + }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + let requests = requests.lock().unwrap().clone(); + (response, requests) +} + fn chat_request() -> LlmRequest { LlmRequest { headers: Map::from_iter([ @@ -96,6 +180,55 @@ fn chat_chunk(model: &str, text: &str) -> Json { }) } +fn responses_response(model: &str, text: &str) -> Json { + json!({ + "id": "resp-test", + "object": "response", + "status": "completed", + "model": model, + "output": [{ + "type": "message", + "id": "msg-test", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }) +} + +fn anthropic_response(model: &str, text: &str) -> Json { + json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }) +} + +fn request_for(protocol: WireProtocol) -> LlmRequest { + let mut request = chat_request(); + request.content = match protocol { + WireProtocol::OpenaiChat => request.content, + WireProtocol::OpenaiResponses => json!({ + "model": "caller/model", + "input": [{ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}] + }] + }), + WireProtocol::AnthropicMessages => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), + }; + request +} + fn provider_error(class: UpstreamFailureClass, status: Option) -> FlowError { FlowError::Upstream(UpstreamFailure { status, @@ -127,6 +260,53 @@ fn random_configuration_rejects_invalid_targets_and_weights() { assert!(validate_config(&config).is_err()); } +#[test] +fn classifier_configuration_validates_targets_thresholds_and_translation_support() { + assert!(validate_config(&classifier_config(false)).is_ok()); + + let mut config = classifier_config(false); + config.targets.remove("classifier"); + assert!( + validate_config(&config) + .unwrap_err() + .contains("algorithm target \"classifier\" is not configured") + ); + + let mut config = classifier_config(false); + if let AlgorithmConfig::LlmClassifier { base_threshold, .. } = &mut config.algorithm { + *base_threshold = 1.5; + } + assert!( + validate_config(&config) + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); + + let mut config = classifier_config(false); + let classifier = config.targets.get_mut("classifier").unwrap(); + classifier.protocol = WireProtocol::AnthropicMessages; + classifier.endpoint = WireProtocol::AnthropicMessages.endpoint().into(); + assert!( + validate_config(&config) + .unwrap_err() + .contains("structured response format") + ); + + let mut config = classifier_config(false); + if let AlgorithmConfig::LlmClassifier { + message_hash_fallback, + .. + } = &mut config.algorithm + { + *message_hash_fallback = true; + } + assert!( + validate_config(&config) + .unwrap_err() + .contains("message_hash_fallback requires session_affinity") + ); +} + #[tokio::test] async fn run_stream_requires_relay_to_fulfill_the_provider_call() { let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); @@ -156,6 +336,273 @@ async fn run_stream_requires_relay_to_fulfill_the_provider_call() { assert!(matches!(returned, Step::ReturnToAgent(_))); } +#[tokio::test] +async fn classifier_run_stream_offloads_the_judge_and_routed_calls_to_relay() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let request = runtime + .libsy_request(WireProtocol::OpenaiChat, &chat_request(), false) + .unwrap(); + let mut steps = runtime + .algorithm + .clone() + .run_stream(LibsyContext::default(), request); + + let Step::CallLlm(classifier) = steps.next().await.unwrap().unwrap() else { + panic!("expected classifier CallLlm before the routing decision"); + }; + assert_eq!(classifier.get_decision().selected_model(), "classifier"); + assert!(!classifier.get_decision().is_routed_call()); + assert!( + classifier + .get_request() + .llm_request + .output + .response_format + .is_some() + ); + classifier + .respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response( + Some("provider/classifier".into()), + CLASSIFIER_VERDICT, + )), + metadata: None, + })) + .unwrap(); + + let Step::Decision(decision) = steps.next().await.unwrap().unwrap() else { + panic!("expected final routing decision after the classifier response"); + }; + assert_eq!(decision.selected_model(), "weak"); + assert_eq!(decision.routing_tier(), Some("weak")); + assert!(decision.is_routed_call()); + + let Step::CallLlm(routed) = steps.next().await.unwrap().unwrap() else { + panic!("expected routed CallLlm"); + }; + assert_eq!(routed.get_decision().selected_model(), "weak"); + routed + .respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response(Some("provider/weak".into()), "answer")), + metadata: None, + })) + .unwrap(); + + assert!(matches!( + steps.next().await.unwrap().unwrap(), + Step::ReturnToAgent(_) + )); +} + +#[tokio::test] +async fn buffered_classifier_dispatches_the_judge_then_the_selected_target() { + let (response, requests) = run_buffered_classifier(WireProtocol::OpenaiChat).await; + assert_eq!( + response["choices"][0]["message"]["content"], + "efficient answer" + ); + + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!(models, vec!["provider/classifier", "provider/weak"]); + assert_eq!( + requests[1].content["provider_extension"], + json!({"preserve": true}) + ); +} + +#[tokio::test] +async fn buffered_classifier_supports_an_openai_responses_judge_target() { + let (response, requests) = run_buffered_classifier(WireProtocol::OpenaiResponses).await; + assert_eq!( + response["choices"][0]["message"]["content"], + "efficient answer" + ); + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!( + models, + vec!["provider/classifier-responses", "provider/weak"] + ); +} + +#[tokio::test] +#[ignore = "blocked by LIBSY-GAP-005 in the pinned Switchyard revision"] +async fn classifier_prompt_is_encoded_as_a_system_instruction() { + let (_, chat_requests) = run_buffered_classifier(WireProtocol::OpenaiChat).await; + assert_eq!(chat_requests[0].content["messages"][0]["role"], "system"); + + let (_, responses_requests) = run_buffered_classifier(WireProtocol::OpenaiResponses).await; + assert!(responses_requests[0].content["instructions"].is_string()); +} + +#[tokio::test] +async fn buffered_classifier_routes_responses_and_anthropic_inbound_profiles() { + for (inbound, call_name) in [ + (WireProtocol::OpenaiResponses, "openai.responses"), + (WireProtocol::AnthropicMessages, "anthropic.messages"), + ] { + let mut config = classifier_config(false); + config.targets.insert( + "weak".into(), + binding(inbound, &format!("provider/weak-{}", inbound.label()), 1.0), + ); + config.enabled_inbound_profiles = BTreeSet::from([inbound]); + config.default_targets = ProtocolDefaults::default(); + match inbound { + WireProtocol::OpenaiResponses => { + config.default_targets.openai_responses = "weak".into() + } + WireProtocol::AnthropicMessages => { + config.default_targets.anthropic_messages = "weak".into() + } + WireProtocol::OpenaiChat => unreachable!(), + } + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmExecutionNextFn = Arc::new(move |request| { + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + if model == "provider/classifier" { + return Ok(chat_response(&model, CLASSIFIER_VERDICT)); + } + Ok(match inbound { + WireProtocol::OpenaiResponses => { + assert!(request.content["input"].is_array()); + responses_response(&model, "efficient answer") + } + WireProtocol::AnthropicMessages => { + assert!(request.content["messages"].is_array()); + anthropic_response(&model, "efficient answer") + } + WireProtocol::OpenaiChat => unreachable!(), + }) + }) + }); + + let response = runtime + .execute_buffered(call_name, request_for(inbound), next) + .await + .unwrap(); + assert!( + response.to_string().contains("efficient answer"), + "classifier response for {inbound:?}: {response}" + ); + } +} + +#[tokio::test] +async fn unavailable_classifier_fails_closed_without_retrying_the_judge() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + if model == "provider/classifier" { + return Err(provider_error(UpstreamFailureClass::Connection, None)); + } + Ok(chat_response(&model, "capable answer")) + }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + assert_eq!( + response["choices"][0]["message"]["content"], + "capable answer" + ); + assert_eq!( + *models.lock().unwrap(), + vec!["provider/classifier", "provider/strong"] + ); +} + +#[tokio::test] +async fn classifier_reconsults_on_retry_without_session_affinity() { + let mut config = classifier_config(false); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let weak_calls = Arc::new(AtomicUsize::new(0)); + let weak_call_count = Arc::clone(&weak_calls); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let weak_call_count = Arc::clone(&weak_call_count); + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + if model == "provider/classifier" { + return Ok(chat_response(&model, CLASSIFIER_VERDICT)); + } + if weak_call_count.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error( + UpstreamFailureClass::RetryableStatus, + Some(503), + )); + } + Ok(chat_response(&model, "retried answer")) + }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + assert_eq!( + response["choices"][0]["message"]["content"], + "retried answer" + ); + assert_eq!( + *models.lock().unwrap(), + vec![ + "provider/classifier", + "provider/weak", + "provider/classifier", + "provider/weak" + ] + ); +} + +#[tokio::test] +async fn classifier_session_affinity_skips_repeated_judge_calls() { + let runtime = SwitchyardRuntime::new(classifier_config(true)).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "answer" + }; + Ok(chat_response(&model, text)) + }) + }); + + for _ in 0..2 { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next.clone()) + .await + .unwrap(); + } + assert_eq!( + *models.lock().unwrap(), + vec!["provider/classifier", "provider/weak", "provider/weak"] + ); +} + #[tokio::test] async fn buffered_dispatch_uses_relay_binding_and_preserves_same_protocol_json() { let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); @@ -367,6 +814,120 @@ async fn concurrent_runs_dispatch_independently() { assert_eq!(calls.load(Ordering::SeqCst), 16); } +#[tokio::test] +async fn streaming_classifier_dispatches_streamed_judge_and_routed_calls() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request); + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "streamed answer" + }; + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + &model, text, + ))]))) + }) + }); + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) + .await + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output + .iter() + .filter_map(|item| item.as_ref().ok()) + .any(|event| event["choices"][0]["delta"]["content"] == "streamed answer") + ); + + let requests = requests.lock().unwrap(); + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!(models, vec!["provider/classifier", "provider/weak"]); + assert!( + requests + .iter() + .all(|request| request.content["stream"] == true) + ); + assert!(requests[0].content["response_format"].is_object()); +} + +#[tokio::test] +async fn streaming_classifier_supports_an_openai_responses_judge_target() { + let mut config = classifier_config(false); + config.targets.insert( + "classifier".into(), + binding( + WireProtocol::OpenaiResponses, + "provider/classifier-responses", + 1.0, + ), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request); + if model == "provider/classifier-responses" { + return Ok(LlmJsonStream::new(stream::iter([ + Ok(json!({ + "type": "response.created", + "response": {"id": "resp-test", "model": model} + })), + Ok(json!({ + "type": "response.output_text.delta", + "output_index": 0, + "content_index": 0, + "delta": CLASSIFIER_VERDICT + })), + ]))); + } + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + &model, + "streamed answer", + ))]))) + }) + }); + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) + .await + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output + .iter() + .filter_map(|item| item.as_ref().ok()) + .any(|event| event["choices"][0]["delta"]["content"] == "streamed answer") + ); + let requests = requests.lock().unwrap(); + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!( + models, + vec!["provider/classifier-responses", "provider/weak"] + ); + assert!(requests[0].content["text"]["format"].is_object()); +} + #[tokio::test] async fn streaming_prefetch_retries_before_commit() { let mut config = chat_config(); diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx index 86f2093cd..cbf6b42ae 100644 --- a/docs/configure-plugins/switchyard/about.mdx +++ b/docs/configure-plugins/switchyard/about.mdx @@ -13,9 +13,10 @@ SPDX-License-Identifier: Apache-2.0 */} > library contract may change before the compatible Switchyard 0.2 crates are > published. -The Switchyard component runs libsy's random router inside NeMo Relay. Relay -drives `Algorithm::run_stream`, performs every provider call, and sends the real -response or error back through `CallLlmRequest::respond`. +The Switchyard component runs libsy's weighted random router or LLM classifier +inside NeMo Relay. Relay drives `Algorithm::run_stream`, performs every +provider call, and sends the real response or error back through +`CallLlmRequest::respond`. There is no `switchyard-server`, Decision API, service health check, Switchyard LLM client, or dedicated Switchyard ATOF receiver. @@ -47,7 +48,7 @@ flowchart LR | Owner | Responsibilities | | --- | --- | -| libsy | Random target selection, decision emission, algorithm lifecycle, and consuming the response supplied by Relay. | +| libsy | Random selection or classifier consultation and tier selection, decision emission, algorithm lifecycle, and consuming each response supplied by Relay. | | `switchyard-translation` | Decode and encode OpenAI Chat, OpenAI Responses, and Anthropic Messages requests, buffered responses, and stream events. | | NeMo Relay | Provider bindings, credentials, URL and endpoint selection, physical dispatch, retries, trusted fallback, scopes, and observability. | @@ -68,5 +69,10 @@ examples/switchyard/run-real-e2e.sh The example starts Relay and one fake provider, exercises deterministic routing between two semantic targets, and validates buffered and streaming fidelity. +The classifier follows the same architecture but produces two provider-call +steps: a classifier consultation followed by the selected weak or strong +target. Both calls use Relay's configured target bindings and +`switchyard-translation`. + Continue with [Switchyard Configuration](./configuration.mdx) for the complete version-2 schema. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx index d61fd937c..531b28600 100644 --- a/docs/configure-plugins/switchyard/configuration.mdx +++ b/docs/configure-plugins/switchyard/configuration.mdx @@ -8,8 +8,8 @@ position: 2 SPDX-License-Identifier: Apache-2.0 */} The Switchyard component runs libsy in process. It currently supports the -random router in enforce mode for OpenAI Chat, OpenAI Responses, and Anthropic -Messages managed LLM calls. +weighted random router and LLM classifier in enforce mode for OpenAI Chat, +OpenAI Responses, and Anthropic Messages managed LLM calls. ## Complete Example @@ -79,7 +79,7 @@ weight = 0 | `priority` | `0` | Relay LLM execution-intercept priority. | | `max_retries` | `3` | Provider retries after the initial libsy run. Maximum: `10`. | | `enabled_inbound_profiles` | All three protocols | Managed Relay profiles intercepted by this component. | -| `algorithm` | Random without a seed | libsy algorithm configuration. Only `kind = "random"` is supported. | +| `algorithm` | Random without a seed | libsy algorithm configuration. Supports `kind = "random"` and `kind = "llm_classifier"`. | | `targets` | Required | Semantic target names mapped to Relay-owned provider bindings. | | `default_targets` | Required per enabled profile | Same-protocol targets used for trusted fallback. | @@ -89,6 +89,40 @@ weight = 0 nonnegative `weight` participates in libsy's weighted selection; `0` excludes a target from normal routing while retaining it as a trusted fallback. +### LLM Classifier Algorithm + +The classifier consults one model, evaluates its structured verdict, and then +routes the original request to either the weak or strong target: + +```toml +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 +capability_elevated_floor = 0.8 +session_affinity = false +message_hash_fallback = false +``` + +| Field | Default | Description | +| --- | --- | --- | +| `classifier_target` | Required | Semantic target for the classifier consultation. Must use `openai_chat` or `openai_responses` under the pinned lossless translation contract. | +| `weak_target` | Required | Efficient target selected when the verdict meets the configured thresholds. | +| `strong_target` | Required | Capable target selected for invalid, abstained, unavailable, or below-threshold verdicts. | +| `base_threshold` | Required | Lowest classifier `p_solve` that selects the weak target. Range: `0.0`–`1.0`. | +| `min_confidence` | `0.0` | Lowest classifier confidence that permits weak-target routing. | +| `capability_elevated_floor` | None | Higher `p_solve` threshold for uncertain, unsupported, or unmatched capability verdicts. | +| `session_affinity` | `false` | Reuse the first decision for later requests carrying the same session identity. | +| `message_hash_fallback` | `false` | Use the first user-message hash when no session identity is available. Requires `session_affinity`. | + +Both the classifier consultation and the selected target call are ordinary +`CallLlm` steps fulfilled by Relay. The classifier target uses Relay's normal +binding, authentication, dispatch, and Switchyard translation path; the plugin +does not use `switchyard-llm-client`. + ### Target Bindings | Field | Required | Description | @@ -124,9 +158,12 @@ engine. The component does not invoke NeMo Relay codecs for provider conversion. ## Retry and Fallback -Every retry starts a new libsy run and obtains a fresh random decision. Relay -returns the actual provider error to libsy before classifying the original typed -Relay error for retry. +Every retry starts a new libsy run. Random routing obtains a fresh selection. +The classifier runs again unless session affinity already retained the routing +decision. Relay returns the actual provider error to libsy before classifying +the original typed Relay error for retry. A failed classifier consultation is +handled by libsy's fail-closed policy and routes to the strong target; it does +not cause Relay to retry the judge. Streaming retries are allowed only before the first provider event. Once the first event commits the stream, later errors are delivered to the caller and do @@ -135,6 +172,15 @@ not trigger another provider call. When attempts are exhausted or a non-retryable routing or translation error occurs, Relay dispatches the configured same-protocol fallback exactly once. +## Observability + +Relay emits `switchyard.routing.requested`, `switchyard.routing.call`, +`switchyard.routing.decision`, `switchyard.routing.retry`, +`switchyard.routing.error`, and `switchyard.routing.fallback` mark events from +real libsy lifecycle data. The `algorithm` field uses libsy's algorithm name. +Classifier call marks set `is_routed_call = false`; the selected weak or strong +call and the final decision set it to `true`. + ## Current Limits - No decision-only or observe-only mode. @@ -142,5 +188,7 @@ occurs, Relay dispatches the configured same-protocol fallback exactly once. - No Switchyard-owned provider client, service, or ATOF ingestion. - No synthesized decision IDs, confidence, baselines, or optimization contributions. +- Streaming traffic requires the configured classifier target to support + streamed structured-output responses. - Provider features that cannot be represented losslessly across protocols are rejected. diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md index eec70cdd0..3a31f5d2f 100644 --- a/examples/switchyard/README.md +++ b/examples/switchyard/README.md @@ -17,20 +17,31 @@ From the Relay repository root: ```bash examples/switchyard/run-real-e2e.sh +examples/switchyard/run-classifier-e2e.sh ``` -The script sends buffered and streaming OpenAI Chat requests through a seeded, -weighted libsy random router. It verifies that both configured provider models -are selected, that Relay performs every physical request, and that a +The first script sends buffered and streaming OpenAI Chat requests through a +seeded, weighted libsy random router. It verifies that both configured provider +models are selected, that Relay performs every physical request, and that a provider-specific `system_fingerprint` survives the same-protocol stream round -trip. +trip. The second script verifies the classifier consultation, weak-tier +selection, structured response format, and buffered and streaming final calls. The files are: -- `plugins.toml`: version-2 library-only Switchyard configuration; +- `plugins.toml`: version-2 weighted-random configuration; +- `classifier-plugins.toml`: version-2 LLM-classifier configuration using the + same Relay-owned provider bindings; - `fake_upstream.py`: deterministic OpenAI-compatible provider; - `e2e-common.sh`: process and readiness helpers; -- `run-real-e2e.sh`: executable no-service smoke test. +- `run-real-e2e.sh`: weighted-random no-service smoke test; +- `run-classifier-e2e.sh`: classifier no-service smoke test. -`plugins.toml` contains no credentials. For real providers, use each target's -`header_env` map and keep secret values outside tracked configuration. +The example configurations contain no credentials. For real providers, use +each target's `header_env` map and keep secret values outside tracked +configuration. + +The classifier configuration demonstrates the two-call `run_stream` lifecycle: +the fake `provider/classifier` returns a structured verdict, then libsy selects +`provider/fast`. Both calls are made by Relay; no Switchyard client or service +is involved. diff --git a/examples/switchyard/classifier-plugins.toml b/examples/switchyard/classifier-plugins.toml new file mode 100644 index 000000000..7f932994a --- /dev/null +++ b/examples/switchyard/classifier-plugins.toml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +version = 2 +priority = 0 +max_retries = 2 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.default_targets] +openai_chat = "quality" + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" + +[components.config.targets.fast] +model = "provider/fast" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" + +[components.config.targets.quality] +model = "provider/quality" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" diff --git a/examples/switchyard/fake_upstream.py b/examples/switchyard/fake_upstream.py index 211cb0472..bc5b5e1d6 100644 --- a/examples/switchyard/fake_upstream.py +++ b/examples/switchyard/fake_upstream.py @@ -8,6 +8,19 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +CLASSIFIER_VERDICT = json.dumps( + { + "recommended_route": "efficient", + "p_solve": 0.9, + "confidence": 0.95, + "abstain": False, + "capability_boundary": "supported", + "primary_rule": "SUP-1", + "crux": "bounded task", + }, + separators=(",", ":"), +) + class Handler(BaseHTTPRequestHandler): log_path: Path @@ -18,6 +31,7 @@ def do_POST(self) -> None: # noqa: N802 with self.log_path.open("a", encoding="utf-8") as output: output.write(json.dumps({"path": self.path, "body": body}) + "\n") model = body.get("model", "fake-model") + content = CLASSIFIER_VERDICT if model == "provider/classifier" else "fake" if body.get("stream"): chunks = [ { @@ -28,7 +42,7 @@ def do_POST(self) -> None: # noqa: N802 "choices": [ { "index": 0, - "delta": {"role": "assistant", "content": "fake"}, + "delta": {"role": "assistant", "content": content}, "finish_reason": None, } ], @@ -58,7 +72,7 @@ def do_POST(self) -> None: # noqa: N802 "choices": [ { "index": 0, - "message": {"role": "assistant", "content": "fake"}, + "message": {"role": "assistant", "content": content}, "finish_reason": "stop", } ], diff --git a/examples/switchyard/run-classifier-e2e.sh b/examples/switchyard/run-classifier-e2e.sh new file mode 100755 index 000000000..4c557bf91 --- /dev/null +++ b/examples/switchyard/run-classifier-e2e.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$relay_root/examples/switchyard/e2e-common.sh" +work_dir="$(mktemp -d)" +upstream_log="$work_dir/upstream.jsonl" + +cleanup() { + local status=$? + e2e_stop_processes + if [[ $status -eq 0 ]]; then + rm -rf "$work_dir" + else + echo "E2E logs preserved in $work_dir" >&2 + e2e_tail_logs "$work_dir" + fi +} +trap cleanup EXIT + +for dependency in cargo curl python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done + +python3 "$relay_root/examples/switchyard/fake_upstream.py" \ + --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & +e2e_add_pid "$!" + +( + cd "$relay_root" + cargo run -p nemo-relay-cli --features switchyard -- \ + --plugin-config-path "$relay_root/examples/switchyard/classifier-plugins.toml" \ + --bind 127.0.0.1:4041 +) >"$work_dir/relay.log" 2>&1 & +relay_pid="$!" +e2e_add_pid "$relay_pid" + +e2e_wait_for http://127.0.0.1:4041/healthz 240 0.25 "$relay_pid" + +curl --fail --silent http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: classifier-buffered' \ + --data-binary \ + '{"model":"client/model","stream":false,"messages":[{"role":"user","content":"classify this"}]}' \ + >"$work_dir/buffered.json" + +curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: classifier-streaming' \ + --data-binary \ + '{"model":"client/model","stream":true,"messages":[{"role":"user","content":"classify this stream"}]}' \ + >"$work_dir/stream.sse" + +python3 - "$upstream_log" "$work_dir/buffered.json" "$work_dir/stream.sse" <<'PY' +import json +import pathlib +import sys + +records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +models = [record["body"]["model"] for record in records] +expected = [ + "provider/classifier", + "provider/fast", + "provider/classifier", + "provider/fast", +] +if models != expected: + raise SystemExit(f"unexpected classifier call sequence: {models}") +for record in records: + body = record["body"] + if body["model"] == "provider/classifier": + if not isinstance(body.get("response_format"), dict): + raise SystemExit(f"classifier request omitted response_format: {body}") +buffered = json.loads(pathlib.Path(sys.argv[2]).read_text()) +if buffered["model"] != "provider/fast": + raise SystemExit(f"classifier did not select the weak target: {buffered}") +stream = pathlib.Path(sys.argv[3]).read_text() +if "fake" not in stream or "[DONE]" not in stream: + raise SystemExit(f"unexpected SSE output: {stream}") +print(f"in-process Switchyard classifier E2E passed: {models}") +PY From 1ec62fb1e4b65f9a475f613210524e1d359988e3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 29 Jul 2026 19:59:48 -0600 Subject: [PATCH 3/5] chore: refresh Rust attributions Signed-off-by: Bryan Bednarski --- ATTRIBUTIONS-Rust.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index b580775eb..c57a7e976 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -24920,8 +24920,9 @@ SOFTWARE. ## md-5 - 0.10.6 **Repository URL**: https://github.com/RustCrypto/hashes -**License Type(s)**: Apache-2.0 -### License: https://spdx.org/licenses/Apache-2.0.html +**License Type(s)**: MIT OR Apache-2.0 +### License: https://spdx.org/licenses/ +### License File: LICENSE-APACHE ``` Apache License Version 2.0, January 2004 @@ -25124,7 +25125,37 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +``` + +### License File: LICENSE-MIT +``` +Copyright (c) 2006-2009 Graydon Hoare +Copyright (c) 2009-2013 Mozilla Foundation +Copyright (c) 2016 Artyom Pavlov + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. ``` ## memchr - 2.8.0 From f22bd65129a0877ce94949003347c2b6c057fcb1 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 09:42:16 -0600 Subject: [PATCH 4/5] fix(switchyard): activate classifier system instructions Signed-off-by: Bryan Bednarski --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- crates/cli/tests/switchyard_process_e2e.rs | 2 ++ crates/switchyard/LIBSY_GAPS.md | 18 ++++++++++-------- crates/switchyard/src/component.rs | 1 + .../switchyard/tests/unit/component_tests.rs | 1 - 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fc888200..90417b298 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3330,7 +3330,7 @@ dependencies = [ [[package]] name = "switchyard-libsy" version = "0.1.0" -source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" dependencies = [ "async-trait", "futures", @@ -3349,7 +3349,7 @@ dependencies = [ [[package]] name = "switchyard-protocol" version = "0.1.0" -source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" dependencies = [ "async-trait", "futures", @@ -3361,7 +3361,7 @@ dependencies = [ [[package]] name = "switchyard-translation" version = "0.1.0" -source = "git+https://github.com/bbednarski9/Switchyard.git?rev=773a451125231e69209cadb697666a1f03d8f909#773a451125231e69209cadb697666a1f03d8f909" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" dependencies = [ "async-stream", "futures", diff --git a/Cargo.toml b/Cargo.toml index bb29ad13e..a227973cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,9 +35,9 @@ nemo-relay-worker = { version = "0.6.0", path = "crates/worker" } nemo-relay-adaptive = { version = "0.6.0", path = "crates/adaptive" } nemo-relay-pii-redaction = { version = "0.6.0", path = "crates/pii-redaction" } nemo-relay-switchyard = { version = "0.6.0", path = "crates/switchyard" } -switchyard-libsy = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } -switchyard-protocol = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } -switchyard-translation = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "773a451125231e69209cadb697666a1f03d8f909" } +switchyard-libsy = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } +switchyard-protocol = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } +switchyard-translation = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } nemo-relay-ffi = { version = "0.6.0", path = "crates/ffi" } nemo-relay-cli = { version = "0.6.0", path = "crates/cli" } opentelemetry = { version = "0.31", default-features = false } diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs index ba0411dfe..9c79532f1 100644 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ b/crates/cli/tests/switchyard_process_e2e.rs @@ -399,6 +399,8 @@ base_url = "{provider_url}" .filter(|(_, body)| body["model"] == "provider/classifier") { assert!(body["response_format"].is_object()); + assert_eq!(body["messages"][0]["role"], "system"); + assert_eq!(body["messages"][1]["role"], "user"); } drop(providers); diff --git a/crates/switchyard/LIBSY_GAPS.md b/crates/switchyard/LIBSY_GAPS.md index 04b0afdab..b49b10827 100644 --- a/crates/switchyard/LIBSY_GAPS.md +++ b/crates/switchyard/LIBSY_GAPS.md @@ -114,16 +114,18 @@ dedicated model-unavailable category. - **Relay test unblocked:** A contract test proving that model-unavailable reaches an algorithm as a distinct typed failure. -## LIBSY-GAP-005: Classifier Prompt Instruction Role +## LIBSY-GAP-005: Classifier Prompt Instruction Role (Resolved) **Description:** The LLM classifier's governing prompt must reach the provider as a system-level instruction, not as ordinary user content. +- **Status:** Resolved by + [NVIDIA-NeMo/Switchyard#195](https://github.com/NVIDIA-NeMo/Switchyard/pull/195). - **Priority:** P0 for classifier routing correctness. - **Affected Relay behavior:** Relay dispatches the classifier `CallLlm` exactly as libsy constructs it. It must not rewrite neutral message roles around the Switchyard translation contract. -- **Pinned behavior:** `CapabilityJudge` inserts its prompt as +- **Original pinned behavior:** `CapabilityJudge` inserted its prompt as `Message { role: System }`, while the OpenAI Chat encoder maps system roles found in the normal message list to `user`. The classifier therefore receives its policy prompt at the wrong privilege level. @@ -135,17 +137,17 @@ as a system-level instruction, not as ordinary user content. then map it to the provider's native system-instruction field. - **Proposed API area:** `switchyard-libsy::algorithms::LlmTaskClassifier` request construction; no Relay API or codec change. -- **Upstream work:** `feat/libsy-classifier-system-instruction`. -- **Switchyard issue/PR:** Pending focused upstream review. The Relay - development pin will include the resolving commit alongside the stream - preservation commit until compatible Switchyard 0.2 crates are published. +- **Switchyard issue/PR:** + [NVIDIA-NeMo/Switchyard#195](https://github.com/NVIDIA-NeMo/Switchyard/pull/195), + now included in the development pin because Switchyard PR #192 was rebased + after the fix merged. Compatible versioned dependencies will replace the + source pin after Switchyard publishes them. - **Acceptance:** The libsy classifier contract test observes one system instruction and one user task message; Relay's focused contract probe observes the provider-native system field for both supported classifier protocols. - **Relay test unblocked:** - `classifier_prompt_is_encoded_as_a_system_instruction` (ignored while Relay - is pinned to the affected Switchyard revision). + `classifier_prompt_is_encoded_as_a_system_instruction` is active and passing. ## Deferred Capabilities diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs index c5672d62a..bdfc9e127 100644 --- a/crates/switchyard/src/component.rs +++ b/crates/switchyard/src/component.rs @@ -1016,6 +1016,7 @@ fn build_algorithm(config: &SwitchyardConfig) -> Result, Stri capability_elevated_floor: *capability_elevated_floor, session_affinity: *session_affinity, message_hash_fallback: *message_hash_fallback, + recent_turn_window: None, }, ) .map(|algorithm| Arc::new(algorithm) as Arc) diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs index d17b6c4de..c89321aa4 100644 --- a/crates/switchyard/tests/unit/component_tests.rs +++ b/crates/switchyard/tests/unit/component_tests.rs @@ -431,7 +431,6 @@ async fn buffered_classifier_supports_an_openai_responses_judge_target() { } #[tokio::test] -#[ignore = "blocked by LIBSY-GAP-005 in the pinned Switchyard revision"] async fn classifier_prompt_is_encoded_as_a_system_instruction() { let (_, chat_requests) = run_buffered_classifier(WireProtocol::OpenaiChat).await; assert_eq!(chat_requests[0].content["messages"][0]["role"], "system"); From 44438179c37cfb214901fd80c126605e075f3935 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 30 Jul 2026 09:42:45 -0600 Subject: [PATCH 5/5] chore: refresh Rust attributions Signed-off-by: Bryan Bednarski --- ATTRIBUTIONS-Rust.md | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index c57a7e976..b580775eb 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -24920,9 +24920,8 @@ SOFTWARE. ## md-5 - 0.10.6 **Repository URL**: https://github.com/RustCrypto/hashes -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -25125,37 +25124,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -Copyright (c) 2006-2009 Graydon Hoare -Copyright (c) 2009-2013 Mozilla Foundation -Copyright (c) 2016 Artyom Pavlov - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## memchr - 2.8.0