diff --git a/ATTRIBUTIONS-Rust.md b/ATTRIBUTIONS-Rust.md index 2821c2c15..b580775eb 100644 --- a/ATTRIBUTIONS-Rust.md +++ b/ATTRIBUTIONS-Rust.md @@ -24920,9 +24920,8 @@ SOFTWARE. ## md-5 - 0.10.6 **Repository URL**: https://github.com/RustCrypto/hashes -**License Type(s)**: MIT OR Apache-2.0 -### License: https://spdx.org/licenses/ -### License File: LICENSE-APACHE +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -25125,37 +25124,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -``` - -### License File: LICENSE-MIT -``` -Copyright (c) 2006-2009 Graydon Hoare -Copyright (c) 2009-2013 Mozilla Foundation -Copyright (c) 2016 Artyom Pavlov - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. ``` ## memchr - 2.8.0 @@ -28444,6 +28413,87 @@ limitations under the License. ``` +## opentelemetry - 0.32.0 +**Repository URL**: https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + ## opentelemetry-http - 0.31.0 **Repository URL**: https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-http **License Type(s)**: Apache-2.0 @@ -34023,7 +34073,7 @@ limitations under the License. ``` -## rand - 0.9.3 +## rand - 0.8.7 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html @@ -34104,7 +34154,7 @@ limitations under the License. ``` -## rand_chacha - 0.9.0 +## rand - 0.9.3 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html @@ -34185,8 +34235,8 @@ limitations under the License. ``` -## rand_core - 0.10.1 -**Repository URL**: https://github.com/rust-random/rand_core +## rand_chacha - 0.3.1 +**Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html ``` @@ -34378,12 +34428,107 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + ``` -## rand_core - 0.9.5 +## rand_chacha - 0.9.0 **Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + +## rand_core - 0.10.1 +**Repository URL**: https://github.com/rust-random/rand_core +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html ``` Apache License Version 2.0, January 2004 @@ -34575,34 +34720,424 @@ APPENDIX: How to apply the Apache License to your work. ``` -## rcgen - 0.13.2 -**Repository URL**: https://github.com/rustls/rcgen +## rand_core - 0.6.4 +**Repository URL**: https://github.com/rust-random/rand **License Type(s)**: Apache-2.0 ### License: https://spdx.org/licenses/Apache-2.0.html ``` -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +``` + +## rand_core - 0.9.5 +**Repository URL**: https://github.com/rust-random/rand +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +``` + +## rcgen - 0.13.2 +**Repository URL**: https://github.com/rustls/rcgen +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." @@ -43391,6 +43926,87 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` +## switchyard-libsy - 0.1.0 +**Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard +**License Type(s)**: Apache-2.0 +### License: https://spdx.org/licenses/Apache-2.0.html +``` +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + ## switchyard-protocol - 0.1.0 **Repository URL**: https://github.com/NVIDIA-NeMo/Switchyard **License Type(s)**: Apache-2.0 diff --git a/Cargo.lock b/Cargo.lock index 1ed2ac773..90417b298 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1614,7 +1614,7 @@ dependencies = [ "nemo-relay-worker-proto", "object_store", "openinference-semantic-conventions", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry-otlp", "opentelemetry_sdk", "reqwest", @@ -1685,7 +1685,7 @@ dependencies = [ "nemo-relay-adaptive", "nemo-relay-pii-redaction", "nemo-relay-switchyard", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry_sdk", "percent-encoding", "rcgen", @@ -1799,17 +1799,16 @@ dependencies = [ name = "nemo-relay-switchyard" version = "0.6.0" dependencies = [ - "async-stream", - "axum", "futures-util", + "http", "nemo-relay", - "reqwest", "schemars", "serde", "serde_json", + "switchyard-libsy", + "switchyard-protocol", "switchyard-translation", "tokio", - "uuid", ] [[package]] @@ -2005,7 +2004,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eea63d12c54e611c8b28e031ac1d6d8778c7f399dc1f661da9260a543676a07d" dependencies = [ - "opentelemetry", + "opentelemetry 0.31.0", ] [[package]] @@ -2028,6 +2027,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "js-sys", +] + [[package]] name = "opentelemetry-http" version = "0.31.0" @@ -2037,7 +2045,7 @@ dependencies = [ "async-trait", "bytes", "http", - "opentelemetry", + "opentelemetry 0.31.0", "reqwest", ] @@ -2048,7 +2056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ "http", - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", @@ -2065,7 +2073,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "opentelemetry", + "opentelemetry 0.31.0", "opentelemetry_sdk", "prost", "tonic", @@ -2081,7 +2089,7 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.31.0", "percent-encoding", "rand 0.9.3", "thiserror 2.0.18", @@ -2546,13 +2554,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2567,6 +2586,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2577,6 +2606,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -3289,22 +3327,44 @@ dependencies = [ "sval_nested", ] +[[package]] +name = "switchyard-libsy" +version = "0.1.0" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" +dependencies = [ + "async-trait", + "futures", + "opentelemetry 0.32.0", + "parking_lot", + "rand 0.8.7", + "serde", + "serde_json", + "switchyard-protocol", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "switchyard-protocol" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80192a3665425e92b4e10df3bff748cc60180cd028ba207bd70fb95df6869d5" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" dependencies = [ + "async-trait", + "futures", "serde", "serde_json", + "thiserror 2.0.18", ] [[package]] name = "switchyard-translation" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c977a683d8921e1ee7f01373162e14cbbfac28ef2a87dde8441c1691b2fed9" +source = "git+https://github.com/bbednarski9/Switchyard.git?rev=aa0a31126cc7ca922a6acdf4f1ef656744698c6c#aa0a31126cc7ca922a6acdf4f1ef656744698c6c" dependencies = [ + "async-stream", + "futures", "serde", "serde_json", "switchyard-protocol", diff --git a/Cargo.toml b/Cargo.toml index c561b1d88..a227973cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,9 @@ nemo-relay-worker = { version = "0.6.0", path = "crates/worker" } nemo-relay-adaptive = { version = "0.6.0", path = "crates/adaptive" } nemo-relay-pii-redaction = { version = "0.6.0", path = "crates/pii-redaction" } nemo-relay-switchyard = { version = "0.6.0", path = "crates/switchyard" } -switchyard-translation = "0.1.0" +switchyard-libsy = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } +switchyard-protocol = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } +switchyard-translation = { git = "https://github.com/bbednarski9/Switchyard.git", rev = "aa0a31126cc7ca922a6acdf4f1ef656744698c6c" } nemo-relay-ffi = { version = "0.6.0", path = "crates/ffi" } nemo-relay-cli = { version = "0.6.0", path = "crates/cli" } opentelemetry = { version = "0.31", default-features = false } diff --git a/crates/cli/src/plugins/editor_model.rs b/crates/cli/src/plugins/editor_model.rs index 5fa842616..df0ecabab 100644 --- a/crates/cli/src/plugins/editor_model.rs +++ b/crates/cli/src/plugins/editor_model.rs @@ -15,7 +15,7 @@ use nemo_relay_adaptive::AdaptiveConfig; use nemo_relay_adaptive::plugin_component::ADAPTIVE_PLUGIN_KIND; use nemo_relay_pii_redaction::component::{PII_REDACTION_PLUGIN_KIND, PiiRedactionConfig}; #[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; +use nemo_relay_switchyard::{AlgorithmConfig, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig}; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::{Map, Value, json}; @@ -52,7 +52,7 @@ impl EditableComponent { Self::NemoGuardrails(_) => "NeMo Guardrails", Self::PiiRedaction(_) => "PII Redaction", #[cfg(feature = "switchyard")] - Self::Switchyard(_) => "Switchyard Decision API", + Self::Switchyard(_) => "Switchyard libsy", } } @@ -1174,20 +1174,19 @@ pub(super) fn pii_redaction_summary(state: &ComponentEditorState bool { - !config.decision_profile_id.is_empty() || !config.targets.is_empty() + !config.targets.is_empty() } #[cfg(feature = "switchyard")] pub(super) fn switchyard_summary(state: &ComponentEditorState) -> String { - let profile = if state.config.decision_profile_id.is_empty() { - "unconfigured" - } else { - state.config.decision_profile_id.as_str() + let algorithm = match &state.config.algorithm { + AlgorithmConfig::Random { .. } => "random", + AlgorithmConfig::LlmClassifier { .. } => "llm classifier", }; format!( - "component {}, profile {}, targets {}", + "component {}, algorithm {}, targets {}", if state.enabled { "enabled" } else { "disabled" }, - profile, + algorithm, state.config.targets.len() ) } diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index a263f0a54..e0db8a56a 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -29,9 +29,7 @@ use nemo_relay::plugin::{ use nemo_relay_adaptive::plugin_component::register_adaptive_component; use nemo_relay_pii_redaction::component::register_pii_redaction_component; #[cfg(feature = "switchyard")] -use nemo_relay_switchyard::{ - register_switchyard_component, validate_switchyard_atof_configuration, -}; +use nemo_relay_switchyard::register_switchyard_component; use reqwest::Client; use serde_json::Value; use subtle::ConstantTimeEq; @@ -894,8 +892,6 @@ pub(crate) enum PluginComponentSetupError { PiiRedaction(String), #[cfg(feature = "switchyard")] Switchyard(String), - #[cfg(feature = "switchyard")] - SwitchyardAtof(String), } impl PluginComponentSetupError { @@ -905,8 +901,6 @@ impl PluginComponentSetupError { Self::PiiRedaction(_) => "PII redaction plugin", #[cfg(feature = "switchyard")] Self::Switchyard(_) => "Switchyard plugin", - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(_) => "Switchyard ATOF", } } @@ -917,8 +911,6 @@ impl PluginComponentSetupError { } #[cfg(feature = "switchyard")] Self::Switchyard(error) => format!("registration failed: {error}"), - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => error.clone(), } } } @@ -939,10 +931,6 @@ impl std::fmt::Display for PluginComponentSetupError { Self::Switchyard(error) => { write!(formatter, "Switchyard plugin registration failed: {error}") } - #[cfg(feature = "switchyard")] - Self::SwitchyardAtof(error) => { - write!(formatter, "Switchyard ATOF validation failed: {error}") - } } } } @@ -961,10 +949,6 @@ pub(crate) fn register_and_validate_plugin_components( if let Err(error) = register_switchyard_component() { errors.push(PluginComponentSetupError::Switchyard(error.to_string())); } - #[cfg(feature = "switchyard")] - if let Err(error) = validate_switchyard_atof_configuration(_plugin_config) { - errors.push(PluginComponentSetupError::SwitchyardAtof(error)); - } errors } diff --git a/crates/cli/tests/switchyard_process_e2e.rs b/crates/cli/tests/switchyard_process_e2e.rs index a9ba3397d..9c79532f1 100644 --- a/crates/cli/tests/switchyard_process_e2e.rs +++ b/crates/cli/tests/switchyard_process_e2e.rs @@ -11,10 +11,12 @@ use axum::body::Body; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::Response; -use axum::routing::{get, post}; +use axum::routing::post; use axum::{Json, Router}; use serde_json::{Value, json}; +const CLASSIFIER_VERDICT: &str = r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.95,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#; + fn gateway_bin() -> &'static str { env!("CARGO_BIN_EXE_nemo-relay") } @@ -28,53 +30,6 @@ impl Drop for ChildGuard { } } -#[derive(Clone, Default)] -struct DecisionState { - requests: Arc>>, -} - -async fn decide( - State(state): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let call = { - let mut requests = state.requests.lock().unwrap(); - requests.push((headers, request)); - requests.len() - }; - if call == 4 { - return Response::builder() - .status(StatusCode::SERVICE_UNAVAILABLE) - .body(Body::from("decision API unavailable")) - .unwrap(); - } - let body = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": format!("decision-{call}"), - "router": {"name": "fake-ci-router", "version": "1"}, - "route": { - "tier": "strong", - "target_model": "provider/selected", - "backend_id": "selected-chat", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.99, - "reason_code": "ci_fixture", - "reason_summary": "deterministic process E2E decision" - }); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(body.to_string())) - .unwrap() -} - -async fn switchyard_health() -> Json { - Json(json!({"status": "ok"})) -} - #[derive(Clone, Default)] struct ProviderState { requests: Arc>>, @@ -89,9 +44,7 @@ async fn provide( let model = request["model"].as_str().unwrap_or("unknown").to_string(); let malformed_response = !stream && model == "provider/selected" - && headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response"); + && request["messages"][0]["content"] == "malformed-response"; state.requests.lock().unwrap().push((headers, request)); if malformed_response { return Response::builder() @@ -101,9 +54,15 @@ async fn provide( .unwrap(); } if stream { + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "streamed" + }; let first = json!({ "id": "chat-ci", "object": "chat.completion.chunk", "model": model, - "choices": [{"index": 0, "delta": {"role": "assistant", "content": "streamed"}, "finish_reason": null}] + "system_fingerprint": "fp_process_e2e", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}, "finish_reason": null}] }); let last = json!({ "id": "chat-ci", "object": "chat.completion.chunk", "model": model, @@ -117,9 +76,14 @@ async fn provide( .body(Body::from(body)) .unwrap(); } + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT.to_string() + } else { + format!("served by {model}") + }; let body = json!({ "id": "chat-ci", "object": "chat.completion", "model": model, - "choices": [{"index": 0, "message": {"role": "assistant", "content": format!("served by {model}")}, "finish_reason": "stop"}], + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} }); Response::builder() @@ -162,17 +126,7 @@ async fn wait_for_gateway(client: &reqwest::Client, url: &str, child: &mut Child } #[tokio::test(flavor = "multi_thread")] -async fn switchyard_plugin_routes_buffered_and_streaming_then_fails_open() { - let decision_state = DecisionState::default(); - let decision_requests = Arc::clone(&decision_state.requests); - let (decision_url, decision_task) = start_server( - Router::new() - .route("/v1/routing/decision", post(decide)) - .route("/health", get(switchyard_health)) - .with_state(decision_state), - ) - .await; - +async fn switchyard_plugin_routes_buffered_and_streaming_without_a_service() { let provider_state = ProviderState::default(); let provider_requests = Arc::clone(&provider_state.requests); let (provider_url, provider_task) = start_server( @@ -192,42 +146,30 @@ kind = "switchyard" enabled = true [components.config] -mode = "enforce" -decision_api_url = "{decision_url}/v1/routing/decision" -decision_profile_id = "ci-process-e2e" -request_materialization = "full_body" -context_mode = "payload_only" -decision_timeout_millis = 1000 +version = 2 max_retries = 0 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] openai_chat = "fallback-chat" -openai_responses = "fallback-responses" -anthropic_messages = "fallback-anthropic" [components.config.targets.selected-chat] model = "provider/selected" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "{provider_url}" +weight = 1 [components.config.targets.fallback-chat] model = "provider/fallback" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "{provider_url}" - -[components.config.targets.fallback-responses] -model = "provider/fallback" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "{provider_url}" - -[components.config.targets.fallback-anthropic] -model = "provider/fallback" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "{provider_url}" +weight = 0 "# ); std::fs::write(&config_path, config).unwrap(); @@ -251,6 +193,7 @@ base_url = "{provider_url}" let send_chat = |request_id: &'static str, stream: bool| { client .post(format!("{gateway_url}/v1/chat/completions")) + .header("authorization", "Bearer caller-secret") .header("x-nemo-relay-session-id", "ci-process-session") .header("x-nemo-relay-request-id", request_id) .header( @@ -261,7 +204,7 @@ base_url = "{provider_url}" .json(&json!({ "model": "client/model", "stream": stream, - "messages": [{"role": "user", "content": "process boundary test"}] + "messages": [{"role": "user", "content": request_id}] })) .send() }; @@ -271,56 +214,18 @@ base_url = "{provider_url}" let buffered: Value = buffered.json().await.unwrap(); assert_eq!(buffered["model"], "provider/selected"); - let translated = client - .post(format!("{gateway_url}/v1/responses")) - .header("x-nemo-relay-session-id", "ci-process-session") - .header("x-nemo-relay-request-id", "translated-request") - .json(&json!({ - "model": "client/model", - "stream": false, - "input": "process boundary response translation" - })) - .send() - .await - .unwrap(); - assert!(translated.status().is_success()); - let translated: Value = translated.json().await.unwrap(); - assert_eq!(translated["object"], "response"); - let streaming = send_chat("stream-request", true).await.unwrap(); assert!(streaming.status().is_success()); let streaming = streaming.text().await.unwrap(); assert!(streaming.contains("streamed")); + assert!(streaming.contains("fp_process_e2e")); assert!(streaming.contains("[DONE]")); - let fallback = send_chat("fallback-request", false).await.unwrap(); - assert!(fallback.status().is_success()); - let fallback: Value = fallback.json().await.unwrap(); - assert_eq!(fallback["model"], "provider/fallback"); - let malformed = send_chat("malformed-response", false).await.unwrap(); assert!(malformed.status().is_success()); let malformed: Value = malformed.json().await.unwrap(); assert_eq!(malformed["model"], "provider/fallback"); - let decisions = decision_requests.lock().unwrap(); - assert_eq!(decisions.len(), 5); - for (headers, body) in decisions.iter() { - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); - assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); - assert_eq!( - headers - .get("x-nemo-relay-session-id") - .unwrap() - .to_str() - .unwrap(), - "ci-process-session" - ); - assert_eq!(body["schema_version"], "switchyard.routing_request.v1"); - assert_eq!(body["decision_profile"]["profile_id"], "ci-process-e2e"); - } - drop(decisions); - let providers = provider_requests.lock().unwrap(); let models = providers .iter() @@ -332,25 +237,12 @@ base_url = "{provider_url}" "provider/selected", "provider/selected", "provider/selected", - "provider/fallback", - "provider/selected", "provider/fallback" ] ); - assert!(providers[1].1["messages"].is_array()); - assert!(providers[1].1.get("input").is_none()); - assert_eq!(providers[1].1["messages"][0]["role"], "user"); - assert_eq!( - providers[1].1["messages"][0]["content"], - "process boundary response translation" - ); let malformed_models = providers .iter() - .filter(|(headers, _)| { - headers - .get("x-nemo-relay-request-id") - .is_some_and(|value| value == "malformed-response") - }) + .filter(|(_, body)| body["messages"][0]["content"] == "malformed-response") .map(|(_, body)| body["model"].as_str().unwrap()) .collect::>(); assert_eq!( @@ -360,8 +252,204 @@ base_url = "{provider_url}" for (headers, _) in providers.iter() { assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-url")); assert!(!headers.contains_key("x-nemo-relay-internal-dispatch-route")); + assert!(!headers.contains_key("authorization")); + } + + provider_task.abort(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn switchyard_plugin_runs_classifier_and_routed_calls_without_a_service() { + let provider_state = ProviderState::default(); + let provider_requests = Arc::clone(&provider_state.requests); + let (provider_url, provider_task) = start_server( + Router::new() + .route("/v1/chat/completions", post(provide)) + .with_state(provider_state), + ) + .await; + + let temp = tempfile::tempdir().unwrap(); + let atof_dir = temp.path().join("atof"); + std::fs::create_dir_all(&atof_dir).unwrap(); + let config_path = temp.path().join("plugins.toml"); + let config = format!( + r#"version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 2 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "{}" +filename = "events.jsonl" +mode = "overwrite" + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +version = 2 +max_retries = 0 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "weak" +strong_target = "strong" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.default_targets] +openai_chat = "strong" + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" + +[components.config.targets.weak] +model = "provider/weak" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" + +[components.config.targets.strong] +model = "provider/strong" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "{provider_url}" +"#, + atof_dir.display() + ); + std::fs::write(&config_path, config).unwrap(); + + let address = unused_address(); + let gateway_url = format!("http://{address}"); + let stderr = std::fs::File::create(temp.path().join("gateway.log")).unwrap(); + let child = Command::new(gateway_bin()) + .arg("--plugin-config-path") + .arg(&config_path) + .arg("--bind") + .arg(address.to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::from(stderr)) + .spawn() + .unwrap(); + let mut gateway = ChildGuard(child); + let client = reqwest::Client::new(); + wait_for_gateway(&client, &gateway_url, &mut gateway.0).await; + + let buffered = client + .post(format!("{gateway_url}/v1/chat/completions")) + .header("x-nemo-relay-session-id", "classifier-buffered") + .json(&json!({ + "model": "client/model", + "messages": [{"role": "user", "content": "classify buffered"}] + })) + .send() + .await + .unwrap(); + assert!(buffered.status().is_success()); + let buffered: Value = buffered.json().await.unwrap(); + assert_eq!(buffered["model"], "provider/weak"); + + let streaming = client + .post(format!("{gateway_url}/v1/chat/completions")) + .header("x-nemo-relay-session-id", "classifier-streaming") + .json(&json!({ + "model": "client/model", + "stream": true, + "messages": [{"role": "user", "content": "classify streaming"}] + })) + .send() + .await + .unwrap(); + assert!(streaming.status().is_success()); + let streaming = streaming.text().await.unwrap(); + assert!(streaming.contains("streamed")); + assert!(streaming.contains("[DONE]")); + + let providers = provider_requests.lock().unwrap(); + let models = providers + .iter() + .map(|(_, body)| body["model"].as_str().unwrap()) + .collect::>(); + assert_eq!( + models, + vec![ + "provider/classifier", + "provider/weak", + "provider/classifier", + "provider/weak" + ] + ); + for (_, body) in providers + .iter() + .filter(|(_, body)| body["model"] == "provider/classifier") + { + assert!(body["response_format"].is_object()); + assert_eq!(body["messages"][0]["role"], "system"); + assert_eq!(body["messages"][1]["role"], "user"); } + drop(providers); + + let events_path = atof_dir.join("events.jsonl"); + let mut events = String::new(); + for _ in 0..40 { + events = std::fs::read_to_string(&events_path).unwrap_or_default(); + if events.matches("switchyard.routing.call").count() >= 4 { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + let events = events + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|event| { + event["name"] + .as_str() + .is_some_and(|name| name.starts_with("switchyard.routing.")) + }) + .collect::>(); + let calls = events + .iter() + .filter(|event| event["name"] == "switchyard.routing.call") + .collect::>(); + assert_eq!(calls.len(), 4, "routing events: {events:?}"); + assert!(calls.iter().all(|event| { + event["data"]["algorithm"] == "llm_task_classifier" + && event["data"]["semantic_target"] + .as_str() + .is_some_and(|target| matches!(target, "classifier" | "weak")) + })); + assert_eq!( + calls + .iter() + .filter(|event| event["data"]["is_routed_call"] == false) + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|event| { + event["name"] == "switchyard.routing.decision" + && event["data"]["semantic_target"] == "weak" + }) + .count(), + 2 + ); - decision_task.abort(); provider_task.abort(); } diff --git a/crates/switchyard/Cargo.toml b/crates/switchyard/Cargo.toml index 7e7466884..77a68ee8f 100644 --- a/crates/switchyard/Cargo.toml +++ b/crates/switchyard/Cargo.toml @@ -7,7 +7,7 @@ version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true -description = "First-party Switchyard Decision API routing plugin for NeMo Relay." +description = "First-party in-process Switchyard libsy routing plugin for NeMo Relay." readme = "README.md" [lints] @@ -21,14 +21,13 @@ schema = ["dep:schemars", "nemo-relay/schema"] nemo-relay.workspace = true serde = { version = "1", features = ["derive"] } serde_json = "1" +switchyard-libsy.workspace = true +switchyard-protocol.workspace = true switchyard-translation.workspace = true schemars = { version = "0.8", optional = true } -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } -tokio = { version = "1", default-features = false, features = ["time"] } -uuid = { workspace = true, features = ["v7"] } +tokio = { version = "1", default-features = false, features = ["rt", "sync"] } futures-util = "0.3" -async-stream = "0.3" +http = "1" [dev-dependencies] -axum = "0.8" tokio = { version = "1", features = ["net", "rt", "macros", "sync", "test-util", "rt-multi-thread", "time"] } diff --git a/crates/switchyard/LIBSY_GAPS.md b/crates/switchyard/LIBSY_GAPS.md new file mode 100644 index 000000000..b49b10827 --- /dev/null +++ b/crates/switchyard/LIBSY_GAPS.md @@ -0,0 +1,158 @@ + + +# libsy Contract Assessment + +This assessment covers the Relay-native random-router and LLM-classifier +integrations driven through `Algorithm::run_stream`. Relay performs provider +calls and returns each real response, stream, or error through +`CallLlmRequest::respond`. + +The initial assessment uses Switchyard revision +`c8ca731adec50c948739a71157ce283ed360ea8a`. Development revisions keep +`switchyard-libsy`, `switchyard-protocol`, and `switchyard-translation` on the +same source revision to preserve their shared type identity. Revision pinning +is dependency maintenance, not a libsy feature gap; compatible Switchyard 0.2 +crate releases will replace the source pins. The current Relay draft follows +the focused stream-preservation work in +[NVIDIA-NeMo/Switchyard#192](https://github.com/NVIDIA-NeMo/Switchyard/pull/192). + +## Integration Contract Under Assessment + +```mermaid +flowchart LR + Caller["Caller wire request"] --> DecodeRequest["switchyard-translation decode"] + DecodeRequest --> Run["libsy run_stream"] + Run --> Decision["Decision"] + Decision --> Call["CallLlm"] + Call --> EncodeTarget["switchyard-translation encode target"] + EncodeTarget --> Dispatch["Relay provider dispatch"] + Dispatch --> DecodeResponse["switchyard-translation decode response or stream"] + DecodeResponse --> Respond["CallLlmRequest.respond"] + Respond --> Final["ReturnToAgent"] + Final --> EncodeCaller["switchyard-translation encode caller"] + EncodeCaller --> Caller +``` + +## Confirmed Baseline + +The pinned library supports the required random-router lifecycle: + +- client-less semantic targets; +- weighted and optionally seeded random selection; +- `Decision → CallLlm → ReturnToAgent` through `run_stream`; +- Relay-hosted buffered and streaming provider calls; +- concurrent independent runs; +- typed libsy orchestration and common provider-client errors; +- buffered request and response translation with exact same-format + preservation. + +The same driver can serve the classifier's two-call lifecycle: a classifier +consultation followed by the selected weak or strong target. The classifier +prompt-role gap below must be resolved upstream before that integration is +ready for review. + +## LIBSY-GAP-003: Exact Raw Stream-Event Preservation + +**Description:** A provider stream event must survive the +Relay → libsy → Relay round trip without losing fields that the neutral chunk +model does not understand. + +- **Priority:** P0 for same-protocol streaming parity. +- **Affected Relay behavior:** Same-protocol responses must preserve provider + extensions such as OpenAI Chat `system_fingerprint`. +- **Pinned behavior:** `switchyard-translation` decodes a raw event into + its normalized `LlmResponseChunk` values and later re-encodes those values. + Unknown fields disappear in this Switchyard translation round trip and + generated defaults can appear. +- **Reproducer:** Route an OpenAI Chat chunk containing a text delta and + `system_fingerprint` through the plugin. The pinned output drops the + fingerprint and adds `created: 0`. +- **Required behavior:** Carry the complete parsed source JSON beside its + normalized children. Replay that JSON when source and target formats match; + use only normalized children for cross-format encoding. +- **Proposed API area:** `switchyard-protocol::LlmResponseChunk` and + `switchyard-translation` stream decode/encode methods. +- **Upstream work:** `feat/translation-preserve-raw-stream-events`. +- **Switchyard issue/PR:** + [NVIDIA-NeMo/Switchyard#192](https://github.com/NVIDIA-NeMo/Switchyard/pull/192). + The Relay draft temporarily pins all three Switchyard crates to that PR's + commit. Updating the revisions after it lands is dependency maintenance, not + a separate libsy gap. +- **Acceptance:** JSON-semantically equivalent same-format events for OpenAI + Chat, OpenAI Responses, and Anthropic Messages; normalized cross-format text + translation; nested stream errors remain typed. +- **Relay test unblocked:** `same_protocol_stream_round_trip_preserves_raw_provider_event`. + +## LIBSY-GAP-004: Complete Provider Error Categories + +**Description:** libsy can represent transport, timeout, context-window, HTTP, +translation, and configuration failures, but the pinned protocol has no +dedicated model-unavailable category. + +- **Priority:** P2 for algorithms that react differently to model availability. +- **Affected Relay behavior:** Relay already has a typed + `ModelUnavailable` provider failure and uses the original Relay error for + retry classification. +- **Pinned behavior:** Relay can return the real failure message to libsy only + through the general client-error variant. +- **Reproducer:** Return Relay's typed `ModelUnavailable` failure from a + `CallLlm`. The host can retain that type for its own retry policy, but + `LlmClientError` can carry it back to the algorithm only as `General`. +- **Required behavior:** Add a typed model-unavailable variant without requiring + hosts to parse strings or invent an HTTP status. +- **Proposed API area:** `switchyard-protocol::LlmClientError`. +- **Switchyard issue/PR:** None; deferred until an algorithm needs to react to + this category. +- **Current integration impact:** Not a random-router blocker. The random router + does not branch on provider error category, and Relay retains the original + typed error for its retry decision. +- **Acceptance:** libsy receives a typed model-unavailable failure and existing + algorithms remain source-compatible. +- **Relay test unblocked:** A contract test proving that model-unavailable + reaches an algorithm as a distinct typed failure. + +## LIBSY-GAP-005: Classifier Prompt Instruction Role (Resolved) + +**Description:** The LLM classifier's governing prompt must reach the provider +as a system-level instruction, not as ordinary user content. + +- **Status:** Resolved by + [NVIDIA-NeMo/Switchyard#195](https://github.com/NVIDIA-NeMo/Switchyard/pull/195). +- **Priority:** P0 for classifier routing correctness. +- **Affected Relay behavior:** Relay dispatches the classifier `CallLlm` + exactly as libsy constructs it. It must not rewrite neutral message roles + around the Switchyard translation contract. +- **Original pinned behavior:** `CapabilityJudge` inserted its prompt as + `Message { role: System }`, while the OpenAI Chat encoder maps system roles + found in the normal message list to `user`. The classifier therefore receives + its policy prompt at the wrong privilege level. +- **Reproducer:** Start `LlmTaskClassifier::run_stream`, inspect its first + `CallLlm`, encode that request as OpenAI Chat, and observe + `messages[0].role == "user"`. +- **Required behavior:** Build the classifier prompt in + `LlmRequest.instructions` with `Role::System`. Switchyard's protocol encoders + then map it to the provider's native system-instruction field. +- **Proposed API area:** `switchyard-libsy::algorithms::LlmTaskClassifier` + request construction; no Relay API or codec change. +- **Switchyard issue/PR:** + [NVIDIA-NeMo/Switchyard#195](https://github.com/NVIDIA-NeMo/Switchyard/pull/195), + now included in the development pin because Switchyard PR #192 was rebased + after the fix merged. Compatible versioned dependencies will replace the + source pin after Switchyard publishes them. +- **Acceptance:** The libsy classifier contract test observes one system + instruction and one user task message; Relay's focused contract probe + observes the provider-native system field for both supported classifier + protocols. +- **Relay test unblocked:** + `classifier_prompt_is_encoded_as_a_system_instruction` is active and passing. + +## Deferred Capabilities + +Decision-only execution, observe-only routing, stage routing, ensemble-specific +concurrent dispatch, structured decision identity, confidence, baselines, and +optimization contributions are outside this integration. They are not mapped +around locally and do not block the random-router or LLM-classifier +`run_stream` contract. diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md index ba50b355f..d64531de2 100644 --- a/crates/switchyard/README.md +++ b/crates/switchyard/README.md @@ -3,123 +3,102 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -[![License](https://img.shields.io/github/license/NVIDIA/NeMo-Relay)](https://github.com/NVIDIA/NeMo-Relay/blob/main/LICENSE) -[![GitHub](https://img.shields.io/badge/github-repo-blue?logo=github)](https://github.com/NVIDIA/NeMo-Relay/) -[![Release](https://img.shields.io/github/v/release/NVIDIA/NeMo-Relay?color=green)](https://github.com/NVIDIA/NeMo-Relay/releases) - # NeMo Relay Switchyard Plugin -`nemo-relay-switchyard` is NeMo Relay's experimental integration -with the [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) -Decision API. It adds routing-aware LLM execution intercepts to the Relay -runtime while preserving Relay ownership of provider credentials, target -bindings, dispatch, retries, fallbacks, and observability. - -NeMo Relay 0.6.0 uses a separately running Switchyard Decision API from the -`topic/nemo-relay-integration` branch. - -Install it from crates.io, or build it from the NeMo Relay source checkout with -the optional CLI feature while the Switchyard Decision API contract and -service/library boundary are still evolving. - -Use the plugin to: +`nemo-relay-switchyard` is NeMo Relay's experimental in-process integration +with [NVIDIA NeMo Switchyard](https://github.com/NVIDIA-NeMo/Switchyard). +It runs libsy's weighted random router or judge-backed LLM classifier inside +Relay through `Algorithm::run_stream`. -- **Route through Switchyard decisions**: Select an exact Relay-owned target - using a versioned Decision API contract. -- **Keep provider protocols stable**: Use Switchyard's translation library for - OpenAI Chat, OpenAI Responses, and Anthropic Messages request and response - translation. -- **Preserve Relay execution semantics**: Keep retries, trusted fallbacks, - credentials, streaming behavior, and optimization accounting in Relay. -- **Support staged rollout**: Run in enforce or observe-only mode with - explicit target bindings and protocol defaults. +Relay owns provider target bindings, credentials, transport, retries, fallback, +and observability. libsy emits routing decisions and requests provider calls; +Relay fulfills each `CallLlm` and returns the real response or error through +`CallLlmRequest::respond`. No Switchyard server or Switchyard LLM client is +used. -## Implementation and Runtime Behavior +## Translation Boundary -The plugin includes the following implementation and runtime behavior: +`switchyard-translation` is the plugin's only provider wire-format translation +engine: -- `SwitchyardConfig`: The typed plugin configuration contract. -- `SwitchyardRuntime`: Buffered and streaming routing intercepts. -- Decision and target validation for exact backend, model, protocol, and - endpoint bindings. -- ATOF-backed or payload-only routing context modes. -- Routing marks and model-routing optimization contributions for Relay's - cumulative accounting pipeline. -- Switchyard-owned protocol translation through the - `switchyard-translation` dependency. +1. decode the caller request into the Switchyard neutral request; +2. encode each routed call for the selected target protocol; +3. decode the provider response or stream before returning it to libsy; +4. encode `ReturnToAgent` into the caller protocol. -## Installation and Source Build +Same-protocol buffered bodies and stream events preserve their original +provider JSON. Cross-protocol routes carry only data represented by the target +protocol and fail when Switchyard reports a lossy conversion. The plugin does +not use NeMo Relay codecs as a translation fallback. -Add the crate from crates.io: +## Configuration -```bash -cargo add nemo-relay-switchyard -``` - -To build the optional CLI integration from a NeMo Relay source checkout: +```toml +version = 1 -```bash -cargo build -p nemo-relay-cli --features switchyard -cargo test -p nemo-relay-switchyard -``` +[[components]] +kind = "switchyard" +enabled = true -The resulting CLI includes the Switchyard component only when the `switchyard` -feature is enabled. A default Relay build does not include this experimental -integration. +[components.config] +version = 2 +priority = 0 +max_retries = 3 +enabled_inbound_profiles = ["openai_chat"] -## Runtime Boundary +[components.config.algorithm] +kind = "random" +seed = 42 -The current integration calls Switchyard's HTTP Decision API at runtime. Relay -does not start or supervise the Switchyard service. For ATOF-backed profiles, -Switchyard also provides the `/v1/atof/events` ingestion and accumulator -runtime. The service must therefore be running before Relay activates the -plugin. Activation performs a bounded request to the service's `/health` -endpoint and fails if it does not return `{"status":"ok"}`. This requirement -applies to enforce and observe-only rollout modes. +[components.config.default_targets] +openai_chat = "fast" -The current service setup is documented in -[`examples/switchyard/README.md`](../../examples/switchyard/README.md), including -the pinned topic-branch commit, local configuration, compatibility smoke test, -and trajectory workflow. +[components.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 -Translation runs in-process through Switchyard's Rust translation library. +[components.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" +``` -## Configuration and Registration +Target-map keys are libsy semantic names. Relay configuration remains +authoritative for the physical model, protocol, endpoint, URL, and credentials. +Version-1 Decision API configuration is rejected with a migration error. -The CLI registers the component when built with `--features switchyard` and -accepts a `[[components]]` entry with `kind = "switchyard"`. A minimal -configuration selects the Decision API and trusted protocol defaults: +For classifier routing, configure one Relay target for the judge plus weak and +strong routed targets: ```toml -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "my-profile" -context_mode = "payload_only" -request_materialization = "summary_only" - -[components.config.default_targets] -openai_chat = "my-openai-target" -openai_responses = "my-responses-target" -anthropic_messages = "my-anthropic-target" +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" ``` -For ATOF-backed profiles, configure an enabled Relay ATOF HTTP stream sink that -has a unique `name`, targets the Switchyard ingestion URL, and uses -environment-referenced authentication headers. Set `atof_endpoint_name` in the -Switchyard component to that name. Local ATOF JSONL output alone does not -populate the Switchyard accumulator. Keep provider and Decision API credentials -outside tracked configuration files. +The classifier consultation and the selected model call are both emitted by +`run_stream` and dispatched by Relay. Under the pinned lossless translation +policy, the classifier target must use OpenAI Chat or OpenAI Responses because +its request requires a structured response format. Weak and strong targets may +use any supported protocol. -## Documentation +Build and test the optional CLI integration with: -For more information, refer to the following resources: +```bash +cargo build -p nemo-relay-cli --features switchyard +cargo test -p nemo-relay-switchyard +``` -- [Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about) -- [Switchyard configuration reference](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/configuration) -- [Switchyard integration examples](../../examples/switchyard/README.md) -- [Switchyard repository](https://github.com/NVIDIA-NeMo/Switchyard) +See [`examples/switchyard`](../../examples/switchyard) for a no-service, +fake-provider smoke test. diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs index e8d5fced8..bdfc9e127 100644 --- a/crates/switchyard/src/component.rs +++ b/crates/switchyard/src/component.rs @@ -1,61 +1,54 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Switchyard plugin configuration and Relay execution integration. +//! Switchyard libsy plugin configuration and Relay execution integration. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::pin::Pin; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; +use std::sync::{Arc, Mutex}; use futures_util::StreamExt; +use http::Uri; +use http::header::{HeaderName, HeaderValue}; use nemo_relay::api::event::{CategoryProfile, DataSchema, EventCategory}; use nemo_relay::api::llm::LlmRequest; -use nemo_relay::api::optimization::record_llm_optimization_contribution; use nemo_relay::api::runtime::{ - LlmExecutionFn, LlmJsonStream, LlmStreamExecutionFn, LlmStreamInner, + LlmExecutionFn, LlmExecutionNextFn, LlmJsonStream, LlmStreamExecutionFn, + LlmStreamExecutionNextFn, task_scope_top, }; use nemo_relay::api::scope::{EmitMarkEventParams, event}; -use nemo_relay::codec::optimization::{ - LlmOptimizationContribution, LlmOptimizationKind, LlmOptimizationModel, - LlmOptimizationModelTransition, -}; use nemo_relay::error::{FlowError, Result as FlowResult}; -use nemo_relay::observability::atof::{AtofEndpointFieldNamePolicy, AtofEndpointTransport}; use nemo_relay::plugin::{ - ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginConfig, PluginError, + ConfigDiagnostic, DiagnosticLevel, Plugin, PluginComponentSpec, PluginError, PluginRegistrationContext, Result as PluginResult, deregister_plugin, register_plugin, }; -use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json, json}; -use uuid::Uuid; +use switchyard_libsy::algorithms::{LlmTaskClassifier, Random, TaskClassifierConfig}; +use switchyard_libsy::{ + Algorithm, CallLlmRequest, Context as LibsyContext, Decision, LibsyError, LlmResponse, + LlmTarget, LlmTargetSet, Request as LibsyRequest, Response as LibsyResponse, Step, +}; +use switchyard_protocol::{LlmClientError, Metadata}; -use crate::contract::{ - DecisionAttempt, DecisionProfile, ROUTING_DECISION_SCHEMA_VERSION, - ROUTING_REQUEST_SCHEMA_VERSION, RequestIdentity, RequestMaterialization, RequestProtocol, - RequestSummary, RoutingDecision, RoutingRequest, RoutingTarget, +use crate::stream_translation::{ + StreamCloseTracker, flow_to_client_error, provider_response_stream, relay_response_stream, }; -use crate::stream_translation::StreamTranscoder; use crate::translation::{ - decode_request, encode_request, latest_user_prompt, recent_message_window, translate_response, - translation_engine, validate_portable_request, + decode_request, decode_response, encode_request, encode_response, translation_engine, + wire_format, }; /// Plugin kind used in Relay plugin configuration. pub const SWITCHYARD_PLUGIN_KIND: &str = "switchyard"; -const SWITCHYARD_HEALTH_PATH: &str = "/health"; -const SWITCHYARD_HEALTH_TIMEOUT: Duration = Duration::from_secs(2); -const SWITCHYARD_HEALTH_MAX_ATTEMPTS: usize = 3; -const SWITCHYARD_HEALTH_INITIAL_BACKOFF: Duration = Duration::from_millis(100); const INTERNAL_DISPATCH_URL_HEADER: &str = "x-nemo-relay-internal-dispatch-url"; const INTERNAL_DISPATCH_ROUTE_HEADER: &str = "x-nemo-relay-internal-dispatch-route"; const INTERNAL_RETRY_AWARE_HEADER: &str = "x-nemo-relay-internal-retry-aware"; +const RELAY_REQUEST_ID_HEADER: &str = "x-nemo-relay-request-id"; +const RELAY_TURN_ID_HEADER: &str = "x-nemo-relay-turn-id"; const ROUTING_MARK_SCHEMA: &str = "switchyard.routing_mark"; -const ROUTING_CONTRIBUTION_SCHEMA: &str = "nvidia.switchyard.routing_optimization"; /// Supported provider wire protocols. #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] @@ -71,7 +64,7 @@ pub enum WireProtocol { } impl WireProtocol { - fn label(self) -> &'static str { + pub(crate) fn label(self) -> &'static str { match self { Self::OpenaiChat => "openai_chat", Self::OpenaiResponses => "openai_responses", @@ -87,7 +80,7 @@ impl WireProtocol { } } - fn from_call(name: &str, request: &LlmRequest) -> Option { + fn from_call(name: &str) -> Option { match name { "openai.chat_completions" | "openai_chat" | "openai_chat_completions" => { Some(Self::OpenaiChat) @@ -96,68 +89,36 @@ impl WireProtocol { "anthropic.messages" | "anthropic" | "anthropic_messages" => { Some(Self::AnthropicMessages) } - _ if request.content.get("input").is_some() => Some(Self::OpenaiResponses), - _ if request.content.get("system").is_some() => Some(Self::AnthropicMessages), - _ if request.content.get("messages").is_some() => Some(Self::OpenaiChat), _ => None, } } } -/// Routing rollout mode. -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RoutingMode { - /// Apply Switchyard decisions. - #[default] - Enforce, - /// Record decisions but dispatch trusted defaults. - ObserveOnly, -} - -impl RoutingMode { - fn label(self) -> &'static str { - match self { - Self::Enforce => "enforce", - Self::ObserveOnly => "observe_only", - } - } -} - -/// Whether the selected Switchyard profile depends on ATOF-derived history. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum ContextMode { - /// The router uses only current request material. - PayloadOnly, - /// Stable identity and a configured ATOF endpoint are required. - AtofRequired, -} - -/// Exact Relay-owned backend binding for one Switchyard backend ID. +/// Exact Relay-owned provider binding for one libsy semantic target. #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct TargetBinding { - /// Exact model expected in the Switchyard decision. + /// Provider model sent on the physical request. pub model: String, - /// Exact protocol expected in the Switchyard decision. + /// Provider wire protocol. pub protocol: WireProtocol, - /// Exact endpoint expected in the Switchyard decision. + /// Provider endpoint. pub endpoint: String, - /// Relay-owned backend base URL. + /// Relay-owned provider base URL. pub base_url: String, - /// Static non-sensitive backend headers. + /// Relative random-routing weight. + #[serde(default = "default_weight")] + pub weight: f64, + /// Static non-sensitive provider headers. #[serde(default)] pub headers: BTreeMap, - /// Backend headers resolved from environment variables. + /// Provider headers resolved from environment variables. #[serde(default)] pub header_env: BTreeMap, } -/// Trusted fallback target IDs for each inbound protocol. -#[derive(Clone, Debug, Deserialize, Serialize)] +/// Trusted fallback target names for each inbound protocol. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct ProtocolDefaults { /// OpenAI Chat fallback target. @@ -181,111 +142,104 @@ impl ProtocolDefaults { } } +/// In-process libsy algorithm configuration. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AlgorithmConfig { + /// Uniform or weighted random routing. + Random { + /// Optional deterministic random seed. + #[serde(default)] + seed: Option, + }, + /// Judge-backed routing between efficient and capable targets. + LlmClassifier { + /// Semantic target used for the classifier consultation. + classifier_target: String, + /// Semantic target used for tasks the classifier considers efficient. + weak_target: String, + /// Semantic target used for tasks requiring the capable tier. + strong_target: String, + /// Lowest solve probability that selects the weak target. + base_threshold: f64, + /// Lowest classifier confidence that permits weak-target routing. + #[serde(default)] + min_confidence: f64, + /// Higher solve-probability floor for uncertain capability verdicts. + #[serde(default)] + capability_elevated_floor: Option, + /// Reuse the first routing decision for subsequent requests in a session. + #[serde(default)] + session_affinity: bool, + /// Key affinity from the first user message when session metadata is absent. + #[serde(default)] + message_hash_fallback: bool, + }, +} + +impl Default for AlgorithmConfig { + fn default() -> Self { + Self::Random { seed: None } + } +} + /// Versioned Switchyard plugin configuration. #[derive(Clone, Debug, Deserialize, Serialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct SwitchyardConfig { - /// Config schema version. + /// Config schema version. The libsy-only contract is version 2. #[serde(default = "default_version")] pub version: u32, - /// Enforce or observe-only rollout mode. - #[serde(default)] - pub mode: RoutingMode, /// Execution-intercept priority. #[serde(default)] pub priority: i32, - /// Switchyard Decision API URL. - pub decision_api_url: String, - /// Switchyard profile ID. - pub decision_profile_id: String, - /// Current-request materialization. - pub request_materialization: RequestMaterialization, - /// Profile context requirement. - pub context_mode: ContextMode, - /// Decision call timeout. - #[serde(default = "default_decision_timeout_millis")] - pub decision_timeout_millis: u64, - /// Provider retries after the initial attempt. + /// Provider retries after the initial libsy run. #[serde(default = "default_max_retries")] pub max_retries: u32, - /// Number of messages in recent-message materialization. - #[serde(default = "default_recent_message_count")] - pub recent_message_count: usize, - /// Static non-sensitive Decision API headers. - #[serde(default)] - pub decision_headers: BTreeMap, - /// Decision API headers resolved from environment variables. + /// In-process libsy algorithm. #[serde(default)] - pub decision_header_env: BTreeMap, - /// Enabled inbound protocols. - #[serde(default = "default_enabled_protocols")] - pub enabled_inbound_profiles: BTreeSet, - /// Exact backend bindings keyed by Switchyard backend ID. + pub algorithm: AlgorithmConfig, + /// Semantic libsy targets mapped to Relay-owned provider bindings. pub targets: BTreeMap, /// Trusted per-protocol fallbacks. - pub default_targets: ProtocolDefaults, - /// Named observability ATOF endpoint used by history-backed profiles. #[serde(default)] - pub atof_endpoint_name: Option, + pub default_targets: ProtocolDefaults, + /// Inbound protocols handled by this component. + #[serde(default = "default_enabled_protocols")] + pub enabled_inbound_profiles: BTreeSet, } impl Default for SwitchyardConfig { fn default() -> Self { Self { version: default_version(), - mode: RoutingMode::default(), priority: 0, - decision_api_url: "http://127.0.0.1:8080/v1/routing/decision".into(), - decision_profile_id: String::new(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: default_decision_timeout_millis(), max_retries: default_max_retries(), - recent_message_count: default_recent_message_count(), - decision_headers: BTreeMap::new(), - decision_header_env: BTreeMap::new(), - enabled_inbound_profiles: default_enabled_protocols(), + algorithm: AlgorithmConfig::default(), targets: BTreeMap::new(), - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: String::new(), - anthropic_messages: String::new(), - }, - atof_endpoint_name: None, + default_targets: ProtocolDefaults::default(), + enabled_inbound_profiles: default_enabled_protocols(), } } } nemo_relay::editor_config! { impl SwitchyardConfig { - mode => { label: "Rollout mode", kind: Enum, values: ["enforce", "observe_only"] }, priority => { label: "Intercept priority", kind: Integer }, - decision_api_url => { label: "Decision API URL", kind: String }, - decision_profile_id => { label: "Decision profile ID", kind: String }, - request_materialization => { - label: "Request materialization", - kind: Enum, - values: ["none", "summary_only", "latest_user_prompt", "recent_message_window", "annotated_request", "full_body"] - }, - context_mode => { label: "Context mode", kind: Enum, values: ["payload_only", "atof_required"] }, - decision_timeout_millis => { label: "Decision timeout (ms)", kind: Integer }, max_retries => { label: "Maximum provider retries", kind: Integer }, - recent_message_count => { label: "Recent message count", kind: Integer }, - decision_headers => { label: "Decision API static headers", kind: StringMap }, - decision_header_env => { label: "Decision API environment headers", kind: StringMap }, - enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json }, - targets => { label: "Backend target bindings", kind: Json }, + algorithm => { label: "libsy algorithm", kind: Json }, + targets => { label: "Provider target bindings", kind: Json }, default_targets => { label: "Trusted protocol defaults", kind: Json }, - atof_endpoint_name => { label: "ATOF endpoint name", kind: String, optional: true } + enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json } } } impl From for PluginComponentSpec { fn from(value: SwitchyardConfig) -> Self { - let Json::Object(config) = - serde_json::to_value(value).expect("Switchyard config should serialize to an object") - else { - unreachable!("Switchyard config must serialize to an object") + let config = match serde_json::to_value(value) { + Ok(Json::Object(config)) => config, + _ => Map::new(), }; Self { kind: SWITCHYARD_PLUGIN_KIND.into(), @@ -296,17 +250,17 @@ impl From for PluginComponentSpec { } fn default_version() -> u32 { - 1 -} -fn default_decision_timeout_millis() -> u64 { - 25 + 2 } + fn default_max_retries() -> u32 { 3 } -fn default_recent_message_count() -> usize { - 8 + +fn default_weight() -> f64 { + 1.0 } + fn default_enabled_protocols() -> BTreeSet { BTreeSet::from([ WireProtocol::OpenaiChat, @@ -351,10 +305,7 @@ impl Plugin for SwitchyardPlugin { .and_then(SwitchyardRuntime::new) .map_err(PluginError::InvalidConfig)?, ); - runtime - .require_healthy_sidecar() - .await - .map_err(PluginError::RegistrationFailed)?; + let buffered = Arc::clone(&runtime); let buffered_intercept: LlmExecutionFn = Arc::new(move |name, request, next| { let runtime = Arc::clone(&buffered); @@ -362,7 +313,7 @@ impl Plugin for SwitchyardPlugin { Box::pin(async move { runtime.execute_buffered(&name, request, next).await }) }); ctx.register_llm_execution_intercept( - "decision", + "libsy", runtime.config.priority, buffered_intercept, )?; @@ -374,7 +325,7 @@ impl Plugin for SwitchyardPlugin { Box::pin(async move { runtime.execute_stream(&name, request, next).await }) }); ctx.register_llm_stream_execution_intercept( - "decision_stream", + "libsy_stream", runtime.config.priority, stream_intercept, )?; @@ -399,939 +350,581 @@ pub fn deregister_switchyard_component() -> bool { deregister_plugin(SWITCHYARD_PLUGIN_KIND) } -/// Validate the cross-component ATOF requirement for enabled history-backed profiles. -pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<(), String> { - let Some(component) = config - .components - .iter() - .find(|component| component.enabled && component.kind == SWITCHYARD_PLUGIN_KIND) - else { - return Ok(()); - }; - let switchyard = parse_config(&component.config)?; - if switchyard.context_mode != ContextMode::AtofRequired { - return Ok(()); - } - let required_name = validate_atof_endpoint_name(switchyard.atof_endpoint_name.as_deref())? - .ok_or_else(|| { - "atof_required Switchyard profiles require atof_endpoint_name".to_string() - })?; - let observability = config - .components - .iter() - .find(|component| component.enabled && component.kind == "observability") - .ok_or_else(|| "atof_required Switchyard profiles require observability".to_string())?; - let sinks = observability - .config - .get("atof") - .filter(|atof| atof.get("enabled").and_then(Json::as_bool) == Some(true)) - .and_then(|atof| atof.get("sinks")) - .and_then(Json::as_array) - .ok_or_else(|| { - "atof_required Switchyard profiles require an enabled ATOF endpoint".to_string() - })?; - let matching_sinks = sinks - .iter() - .filter(|sink| { - sink.get("type").and_then(Json::as_str) == Some("stream") - && sink.get("name").and_then(Json::as_str) == Some(required_name) - }) - .collect::>(); - let endpoint = match matching_sinks.as_slice() { - [sink] => *sink, - [] => { - return Err(format!( - "atof_required Switchyard profile requires named ATOF endpoint {required_name:?}" - )); - } - _ => { - return Err(format!( - "ATOF endpoint name {required_name:?} must resolve to exactly one endpoint" - )); - } - }; - let transport = endpoint.get("transport").map_or_else( - || Some(AtofEndpointTransport::default()), - |value| value.as_str().and_then(AtofEndpointTransport::parse), - ); - if transport != Some(AtofEndpointTransport::HttpPost) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use transport = http_post" - )); - } - let field_name_policy = endpoint.get("field_name_policy").map_or_else( - || Some(AtofEndpointFieldNamePolicy::default()), - |value| value.as_str().and_then(AtofEndpointFieldNamePolicy::parse), - ); - if field_name_policy != Some(AtofEndpointFieldNamePolicy::Preserve) { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} must use field_name_policy = preserve" - )); - } - if endpoint - .get("header_env") - .and_then(Json::as_object) - .is_none_or(Map::is_empty) +fn parse_config(config: &Map) -> Result { + if config.get("version").and_then(Json::as_u64) == Some(1) + || config.contains_key("decision_api_url") { - return Err(format!( - "Switchyard ATOF endpoint {required_name:?} authentication must use at least one environment-referenced header" - )); + return Err( + "Switchyard config version 1 used the removed switchyard-server Decision API; migrate to version = 2 with [components.config.algorithm]".into(), + ); } - Ok(()) -} - -fn parse_config(config: &Map) -> Result { serde_json::from_value(Json::Object(config.clone())) .map_err(|error| format!("invalid Switchyard plugin config: {error}")) } struct SwitchyardRuntime { config: SwitchyardConfig, - client: reqwest::Client, + algorithm: Arc, target_headers: BTreeMap>, translation: switchyard_translation::TranslationEngine, } -enum BufferedAttempt { - Complete(Json), - Retry((String, String)), - Fallback(&'static str), -} - -enum StreamAttempt { - Committed(LlmJsonStream), - Retry((String, String)), - Fallback(&'static str), -} - -struct StreamAttemptContext { - routing_request: RoutingRequest, - decision: RoutingDecision, - attempt: u32, - max_attempts: u32, -} - -fn provider_fallback_reason(error: &FlowError) -> &'static str { - if error_is_retryable(error) { - "retry_exhausted" - } else { - "non_retryable_provider_error" - } -} - impl SwitchyardRuntime { fn new(config: SwitchyardConfig) -> Result { validate_config(&config)?; - let headers = resolve_headers(&config.decision_headers, &config.decision_header_env)?; - let client = reqwest::Client::builder() - .default_headers(headers) - .timeout(Duration::from_millis(config.decision_timeout_millis)) - .build() - .map_err(|error| format!("failed to build Decision API client: {error}"))?; let target_headers = config .targets .iter() - .map(|(id, target)| { - let headers = resolve_json_headers(&target.headers, &target.header_env)?; - Ok((id.clone(), headers)) + .map(|(name, target)| { + resolve_json_headers(&target.headers, &target.header_env) + .map(|headers| (name.clone(), headers)) }) - .collect::>()?; + .collect::>()?; + let algorithm = build_algorithm(&config)?; Ok(Self { config, - client, + algorithm, target_headers, translation: translation_engine(), }) } - // Skip the portability guard when no configured target uses a different protocol: with no - // possible cross-protocol translation, provider-specific fields never need to be portable. - fn may_translate_protocol(&self, inbound: WireProtocol) -> bool { - self.config - .targets - .values() - .any(|target| target.protocol != inbound) - } - - async fn require_healthy_sidecar(&self) -> Result<(), String> { - let health_url = switchyard_health_url(&self.config.decision_api_url)?; - let client = reqwest::Client::builder() - .timeout(SWITCHYARD_HEALTH_TIMEOUT) - .build() - .map_err(|error| format!("failed to build Switchyard health client: {error}"))?; - let mut backoff = SWITCHYARD_HEALTH_INITIAL_BACKOFF; - let mut final_error = None; - for attempt in 1..=SWITCHYARD_HEALTH_MAX_ATTEMPTS { - match check_switchyard_health(&client, &health_url).await { - Ok(()) => return Ok(()), - Err(error) => final_error = Some(error), - } - if attempt < SWITCHYARD_HEALTH_MAX_ATTEMPTS { - tokio::time::sleep(backoff).await; - backoff *= 2; - } - } - Err(final_error.expect("at least one Switchyard health attempt is configured")) - } - async fn execute_buffered( &self, name: &str, original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, + next: LlmExecutionNextFn, ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { + let Some(inbound) = WireProtocol::from_call(name) else { return next(original).await; }; if !self.config.enabled_inbound_profiles.contains(&inbound) { return next(original).await; } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_buffered( - inbound, - original, - next, - "unsupported_provider_extension", - ) - .await; - } - - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_buffered(inbound, original, next, "observe_only") - .await; - } let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; for attempt in 1..=max_attempts { - match self - .buffered_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? - { - BufferedAttempt::Complete(response) => return Ok(response), - BufferedAttempt::Retry(retry) => previous = Some(retry), - BufferedAttempt::Fallback(reason) => { + self.emit_requested(&original, inbound, attempt); + let request = match self.libsy_request(inbound, &original, false) { + Ok(request) => request, + Err(error) => { + self.emit_error( + attempt, + "request_translation", + &error.to_string(), + &original, + ); return self - .dispatch_fallback_buffered(inbound, original, next, reason) + .dispatch_fallback_buffered(inbound, original, next, "translation_error") .await; } - } - } - unreachable!("routing attempt loop always returns") - } - - async fn buffered_attempt( - &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmExecutionNextFn, - attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(BufferedAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - match next(routed).await { - Ok(response) => { - match translate_response(&self.translation, target_protocol, inbound, &response) { - Ok(response) => { - self.record_routing_contribution(&decision, attempt, true); - Ok(BufferedAttempt::Complete(response)) - } + }; + match self + .drive_buffered_run( + request, + dispatch_headers(&original.headers), + next.clone(), + attempt, + ) + .await + { + Ok(response) => match self.finish_buffered(inbound, response) { + Ok(response) => return Ok(response), Err(error) => { self.emit_error( - Some(&routing_request), attempt, "response_translation", &error.to_string(), + &original, ); - Ok(BufferedAttempt::Fallback("translation_error")) + return self + .dispatch_fallback_buffered( + inbound, + original, + next, + "translation_error", + ) + .await; } + }, + Err(failure) if failure.is_retryable() && attempt < max_attempts => { + self.emit_retry(attempt, &failure.error.to_string(), &original); + } + Err(failure) => { + let reason = if failure.is_retryable() { + "retry_exhausted" + } else { + "non_retryable_libsy_error" + }; + self.emit_error(attempt, "libsy_run", &failure.error.to_string(), &original); + return self + .dispatch_fallback_buffered(inbound, original, next, reason) + .await; } - } - Err(error) if error_is_retryable(&error) && attempt < max_attempts => { - let retry_reason = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &retry_reason); - self.emit_retry(&routing_request, &decision, attempt, &retry_reason); - Ok(BufferedAttempt::Retry(( - decision.route.backend_id, - retry_reason, - ))) - } - Err(error) => { - let summary = provider_error_summary(&error); - self.emit_error(Some(&routing_request), attempt, "provider", &summary); - Ok(BufferedAttempt::Fallback(provider_fallback_reason(&error))) } } + Err(FlowError::Internal( + "Switchyard retry loop ended without a result".into(), + )) } async fn execute_stream( &self, name: &str, original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, + next: LlmStreamExecutionNextFn, ) -> FlowResult { - let Some(inbound) = WireProtocol::from_call(name, &original) else { + let Some(inbound) = WireProtocol::from_call(name) else { return next(original).await; }; if !self.config.enabled_inbound_profiles.contains(&inbound) { return next(original).await; } - if self.may_translate_protocol(inbound) - && let Err(error) = validate_portable_request(&self.translation, inbound, &original) - { - self.emit_error( - None, - 0, - "unsupported_provider_extension", - &error.to_string(), - ); - return self - .dispatch_fallback_stream(inbound, original, next, "unsupported_provider_extension") - .await; - } - if self.config.mode == RoutingMode::ObserveOnly { - match self.decided_request(inbound, &original, 1, None).await { - Ok((_, decision, _)) => { - self.record_routing_contribution(&decision, 1, false); - } - Err(error) => self.emit_error(None, 1, "decision_api", &error), - } - return self - .dispatch_fallback_stream(inbound, original, next, "observe_only") - .await; - } let max_attempts = self.config.max_retries.saturating_add(1); - let mut previous = None; for attempt in 1..=max_attempts { + self.emit_requested(&original, inbound, attempt); + let request = match self.libsy_request(inbound, &original, true) { + Ok(request) => request, + Err(error) => { + self.emit_error( + attempt, + "request_translation", + &error.to_string(), + &original, + ); + return self + .dispatch_fallback_stream(inbound, original, next, "translation_error") + .await; + } + }; + let tracker = StreamCloseTracker::default(); match self - .stream_attempt(inbound, &original, &next, attempt, previous, max_attempts) - .await? + .drive_stream_run( + request, + dispatch_headers(&original.headers), + next.clone(), + tracker.clone(), + attempt, + ) + .await { - StreamAttempt::Committed(stream) => return Ok(stream), - StreamAttempt::Retry(retry) => previous = Some(retry), - StreamAttempt::Fallback(reason) => { + Ok(response) => match self.finish_stream(inbound, response, tracker) { + Ok(response) => return Ok(response), + Err(error) => { + self.emit_error( + attempt, + "response_translation", + &error.to_string(), + &original, + ); + return self + .dispatch_fallback_stream(inbound, original, next, "translation_error") + .await; + } + }, + Err(failure) if failure.is_retryable() && attempt < max_attempts => { + self.emit_retry(attempt, &failure.error.to_string(), &original); + } + Err(failure) => { + let reason = if failure.is_retryable() { + "retry_exhausted" + } else { + "non_retryable_libsy_error" + }; + self.emit_error( + attempt, + "libsy_stream_run", + &failure.error.to_string(), + &original, + ); return self .dispatch_fallback_stream(inbound, original, next, reason) .await; } } } - unreachable!("stream routing attempt loop always returns") + Err(FlowError::Internal( + "Switchyard stream retry loop ended without a result".into(), + )) } - async fn stream_attempt( + async fn drive_buffered_run( &self, - inbound: WireProtocol, - original: &LlmRequest, - next: &nemo_relay::api::runtime::LlmStreamExecutionNextFn, + request: LibsyRequest, + headers: Map, + next: LlmExecutionNextFn, attempt: u32, - previous: Option<(String, String)>, - max_attempts: u32, - ) -> FlowResult { - let (routing_request, decision, routed) = match self - .decided_request(inbound, original, attempt, previous) - .await - { - Ok(value) => value, - Err(error) => { - self.emit_error(None, attempt, "decision_api", &error); - return Ok(StreamAttempt::Fallback("decision_error")); - } - }; - let target_protocol = protocol_from_label(&decision.route.target_protocol_profile)?; - let context = StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - }; - match next(routed).await { - Ok(mut upstream) => { - let first = upstream.next().await; - Ok(self.classify_open_stream(inbound, target_protocol, context, upstream, first)) + ) -> Result { + let values = context_values(request.metadata.as_ref(), attempt); + let mark_metadata = libsy_identity_metadata(request.metadata.as_ref()); + let mut context = LibsyContext::default(); + context.values = values.into_iter().collect(); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); + } + Ok(Step::CallLlm(call)) => { + self.emit_call(call.get_decision(), attempt, mark_metadata.clone()); + self.serve_buffered_call( + *call, + headers.clone(), + next.clone(), + Arc::clone(&provider_error), + ) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(RunFailure::new(error, &provider_error)), } - Err(error) => Ok(self.classify_stream_setup_error(context, error)), } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) } - fn classify_open_stream( + async fn drive_stream_run( &self, - inbound: WireProtocol, - target_protocol: WireProtocol, - context: StreamAttemptContext, - upstream: LlmJsonStream, - first: Option>, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - match first { - Some(Ok(first)) => { - self.record_routing_contribution(&decision, attempt, true); - let committed = LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(first)), - upstream, - }); - let output = if target_protocol == inbound { - committed - } else { - translated_stream( - target_protocol, - inbound, - decision.route.target_model.clone(), - committed, + request: LibsyRequest, + headers: Map, + next: LlmStreamExecutionNextFn, + tracker: StreamCloseTracker, + attempt: u32, + ) -> Result { + let values = context_values(request.metadata.as_ref(), attempt); + let mark_metadata = libsy_identity_metadata(request.metadata.as_ref()); + let mut context = LibsyContext::default(); + context.values = values.into_iter().collect(); + let mut steps = self.algorithm.clone().run_stream(context, request); + let provider_error = Arc::new(Mutex::new(None)); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.emit_decision(decision.as_ref(), attempt, mark_metadata.clone()); + } + Ok(Step::CallLlm(call)) => { + self.emit_call(call.get_decision(), attempt, mark_metadata.clone()); + self.serve_stream_call( + *call, + headers.clone(), + next.clone(), + tracker.clone(), + Arc::clone(&provider_error), ) - }; - StreamAttempt::Committed(mark_terminal_stream( - output, - "provider_stream_committed", - self.config.mode.label(), - identity_metadata(&routing_request), - )) - } - Some(Err(error)) if error_is_retryable(&error) && attempt < max_attempts => self - .retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - provider_error_summary(&error), - ), - None if attempt < max_attempts => self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_open", - "empty_stream".into(), - ), - Some(Err(error)) => { - let summary = provider_error_summary(&error); - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_open", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) + .await + .map_err(|error| RunFailure::new(error, &provider_error))?; + } + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(RunFailure::new(error, &provider_error)), } - None => StreamAttempt::Fallback("empty_stream"), } + Err(RunFailure::new( + LibsyError::MissingFinalResponse, + &provider_error, + )) } - fn classify_stream_setup_error( + async fn serve_buffered_call( &self, - context: StreamAttemptContext, - error: FlowError, - ) -> StreamAttempt { - let StreamAttemptContext { - routing_request, - decision, - attempt, - max_attempts, - } = context; - let summary = provider_error_summary(&error); - if error_is_retryable(&error) && attempt < max_attempts { - return self.retry_stream_attempt( - &routing_request, - decision, - attempt, - "provider_stream_setup", - summary, - ); + call: CallLlmRequest, + headers: Map, + next: LlmExecutionNextFn, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let is_routed_call = routed.decision.is_routed_call(); + let result = async { + let metadata = routed.request.metadata.clone(); + let (target_protocol, request) = + self.apply_target(&target_name, routed.request, headers, false)?; + let response = match next(request).await { + Ok(response) => response, + Err(error) => { + if is_routed_call { + remember_provider_error(&provider_error, &error); + } + return Err(flow_to_client_error(error, &target_name)); + } + }; + let response = decode_response(&self.translation, target_protocol, &response) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + Ok(LibsyResponse { + llm_response: LlmResponse::Agg(response), + metadata: Some(response_metadata(metadata.as_ref(), target_protocol)), + }) } - self.emit_error( - Some(&routing_request), - attempt, - "provider_stream_setup", - &summary, - ); - StreamAttempt::Fallback(provider_fallback_reason(&error)) + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) } - fn retry_stream_attempt( + async fn serve_stream_call( &self, - routing_request: &RoutingRequest, - decision: RoutingDecision, - attempt: u32, - error_class: &str, - reason: String, - ) -> StreamAttempt { - if reason != "empty_stream" { - self.emit_error(Some(routing_request), attempt, error_class, &reason); + call: CallLlmRequest, + headers: Map, + next: LlmStreamExecutionNextFn, + tracker: StreamCloseTracker, + provider_error: Arc>>, + ) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let is_routed_call = routed.decision.is_routed_call(); + let result = async { + let metadata = routed.request.metadata.clone(); + let (target_protocol, request) = + self.apply_target(&target_name, routed.request, headers, true)?; + let mut upstream = match next(request).await { + Ok(upstream) => upstream, + Err(error) => { + if is_routed_call { + remember_provider_error(&provider_error, &error); + } + return Err(flow_to_client_error(error, &target_name)); + } + }; + let first = match upstream.next().await { + Some(Ok(first)) => first, + Some(Err(error)) => { + if is_routed_call { + remember_provider_error(&provider_error, &error); + } + return Err(flow_to_client_error(error, &target_name)); + } + None => { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }); + } + }; + let response = provider_response_stream( + upstream, + wire_format(target_protocol), + first, + tracker, + target_name.clone(), + )?; + Ok(LibsyResponse { + llm_response: LlmResponse::Stream(response), + metadata: Some(response_metadata(metadata.as_ref(), target_protocol)), + }) } - self.emit_retry(routing_request, &decision, attempt, &reason); - StreamAttempt::Retry((decision.route.backend_id, reason)) + .await + .map_err(|source| LibsyError::client_call(target_name, source)); + call.respond(result) } - async fn decided_request( + fn libsy_request( &self, inbound: WireProtocol, original: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result<(RoutingRequest, RoutingDecision, LlmRequest), String> { - let request = self.routing_request(inbound, original, attempt, previous)?; - self.emit_requested(&request); - let started = Instant::now(); - let response = self - .client - .post(&self.config.decision_api_url) - .header("x-nemo-relay-session-id", &request.identity.session_id) - .json(&request) - .send() - .await - .map_err(|error| format!("Decision API request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format!("Decision API returned HTTP {status}: {body}")); - } - let decision = response - .json::() - .await - .map_err(|error| format!("Decision API returned invalid JSON: {error}"))?; - self.validate_decision(&decision)?; - if let Some(baseline) = decision.baseline_route.as_ref() - && let Err(error) = self.validate_target(baseline) - { - self.emit_error(Some(&request), attempt, "baseline_binding", &error); - } - let routed = self.apply_target(inbound, original.clone(), &decision)?; - let latency = started.elapsed().as_millis() as u64; - self.emit_decision( - &request, - &decision, - attempt, - self.config.mode == RoutingMode::ObserveOnly, - latency, - ); - Ok((request, decision, routed)) - } - - fn routing_request( - &self, - inbound: WireProtocol, - request: &LlmRequest, - attempt: u32, - previous: Option<(String, String)>, - ) -> Result { - let session = header(request, "x-nemo-relay-session-id"); - let stable_request_id = header(request, "x-nemo-relay-request-id"); - if self.config.context_mode == ContextMode::AtofRequired - && (session.is_none() || stable_request_id.is_none()) - { - return Err("stable session and request identity are required for this profile".into()); - } - let identity_is_stable = session.is_some() && stable_request_id.is_some(); - let synthetic_session = format!("request-{}", Uuid::now_v7()); - let session_id = session.unwrap_or_else(|| synthetic_session.clone()); - let request_id = stable_request_id.unwrap_or_else(|| format!("request-{}", Uuid::now_v7())); - let annotated = decode_request(&self.translation, inbound, request) - .map_err(|error| format!("request translation decode failed: {error}"))?; - let current_request = self.materialize(inbound, request, &annotated)?; - let (previous_route, retry_reason) = previous.unzip(); - Ok(RoutingRequest { - schema_version: ROUTING_REQUEST_SCHEMA_VERSION.into(), - decision_profile: DecisionProfile { - profile_id: self.config.decision_profile_id.clone(), - request_materialization: self.config.request_materialization, - }, - identity: RequestIdentity { - session_id, - request_id, - turn_id: header(request, "x-nemo-relay-turn-id"), - parent_scope_id: header(request, "x-nemo-relay-parent-scope-id"), - root_scope_id: header(request, "x-nemo-relay-root-scope-id"), - harness: header(request, "x-nemo-relay-agent-kind") - .unwrap_or_else(|| "unknown".into()), - source: header(request, "x-nemo-relay-source") - .unwrap_or_else(|| "nemo-relay".into()), - owner_id: header(request, "x-nemo-relay-owner-id"), - quality: header(request, "x-nemo-relay-identity-quality").unwrap_or_else(|| { - if identity_is_stable { - "explicit".into() - } else { - "synthetic".into() - } - }), - }, - protocol: RequestProtocol { - inbound_profile: inbound.label().into(), - inbound_endpoint: inbound.endpoint().into(), - desired_response_profile: inbound.label().into(), - }, - request_summary: RequestSummary { - client_requested_model: request - .content - .get("model") - .and_then(Json::as_str) - .map(ToOwned::to_owned), - prompt_token_estimate: None, - tool_count_in_payload: request - .content - .get("tools") - .and_then(Json::as_array) - .map(|tools| tools.len() as u64), - has_system_prompt: Some( - annotated.instructions.iter().any(|instruction| { - instruction.role == switchyard_translation::Role::System - }) || annotated - .messages - .iter() - .any(|message| message.role == switchyard_translation::Role::System), - ), - }, - current_request, - attempt: DecisionAttempt { - routing_attempt: attempt, - max_routing_attempts: self.config.max_retries.saturating_add(1), - previous_route, - retry_reason, - }, + streaming: bool, + ) -> FlowResult { + let mut llm_request = decode_request(&self.translation, inbound, original)?; + llm_request.stream = streaming; + let mut metadata = relay_metadata(&string_headers(&original.headers)); + metadata.wire_format = Some(wire_format(inbound)); + Ok(LibsyRequest { + llm_request, + raw_request: Some(original.content.clone()), + metadata: Some(metadata), }) } - fn materialize( - &self, - inbound: WireProtocol, - request: &LlmRequest, - annotated: &switchyard_translation::LlmRequest, - ) -> Result, String> { - match self.config.request_materialization { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => Ok(None), - RequestMaterialization::FullBody => Ok(Some(json!({"body": request.content}))), - RequestMaterialization::AnnotatedRequest => Ok(Some(json!({ - "body": request.content, - "annotated_request": annotated, - }))), - RequestMaterialization::LatestUserPrompt => { - let prompt = latest_user_prompt(annotated) - .ok_or_else(|| "latest_user_prompt requires a user message".to_string())?; - let latest = recent_message_window(annotated, 1); - let body = encode_request(&self.translation, inbound, &latest, Map::new()) - .map_err(|error| format!("latest user prompt encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "latest_user_prompt": prompt}))) - } - RequestMaterialization::RecentMessageWindow => { - let window = recent_message_window(annotated, self.config.recent_message_count); - let body = encode_request(&self.translation, inbound, &window, Map::new()) - .map_err(|error| format!("recent window encode failed: {error}"))? - .content; - Ok(Some(json!({"body": body, "annotated_request": window}))) - } - } - } - - fn validate_decision(&self, decision: &RoutingDecision) -> Result<(), String> { - if decision.schema_version != ROUTING_DECISION_SCHEMA_VERSION { - return Err(format!( - "unsupported decision schema {:?}", - decision.schema_version - )); - } - self.validate_target(&decision.route).map(|_| ()) - } - - fn validate_target(&self, target: &RoutingTarget) -> Result<&TargetBinding, String> { - let binding = self - .config - .targets - .get(&target.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", target.backend_id))?; - if binding.model != target.target_model - || binding.protocol.label() != target.target_protocol_profile - || binding.endpoint != target.target_endpoint - { - return Err(format!( - "decision target {:?} does not match its exact Relay binding", - target.backend_id - )); - } - Ok(binding) - } - - fn record_routing_contribution(&self, decision: &RoutingDecision, attempt: u32, applied: bool) { - let Some(contribution) = self.routing_contribution(decision, attempt, applied) else { - return; - }; - let _ = record_llm_optimization_contribution(contribution); - } - - fn routing_contribution( - &self, - decision: &RoutingDecision, - attempt: u32, - applied: bool, - ) -> Option { - let baseline = decision - .baseline_route - .as_ref() - .filter(|baseline| self.validate_target(baseline).is_ok())?; - let mut contribution = LlmOptimizationContribution::new( - SWITCHYARD_PLUGIN_KIND, - LlmOptimizationKind::model_routing(), - ); - contribution.applied = applied; - contribution.model_transition = Some(LlmOptimizationModelTransition { - baseline: Some(LlmOptimizationModel::new(&baseline.target_model)), - effective: Some(LlmOptimizationModel::new(&decision.route.target_model)), - }); - contribution.payload_schema = Some(DataSchema { - name: ROUTING_CONTRIBUTION_SCHEMA.to_string(), - version: "1".to_string(), - }); - contribution.payload = Some(json!({ - "decision_id": decision.decision_id, - "selected_backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "baseline_backend_id": baseline.backend_id, - "baseline_tier": baseline.tier, - "routing_attempt": attempt, - "rollout_mode": self.config.mode.label(), - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - })); - Some(contribution) - } - fn apply_target( &self, - inbound: WireProtocol, - request: LlmRequest, - decision: &RoutingDecision, - ) -> Result { - let binding = self - .config - .targets - .get(&decision.route.backend_id) - .ok_or_else(|| format!("unknown backend_id {:?}", decision.route.backend_id))?; - let annotated = decode_request(&self.translation, inbound, &request) - .map_err(|error| format!("request decode failed: {error}"))?; - let mut routed = if inbound == binding.protocol { - request - } else { - encode_request( - &self.translation, - binding.protocol, - &annotated, - request.headers, - ) - .map_err(|error| format!("request translation failed: {error}"))? - }; - let object = routed - .content - .as_object_mut() - .ok_or_else(|| "translated request body is not an object".to_string())?; - object.insert("model".into(), Json::String(binding.model.clone())); - if let Some(headers) = self.target_headers.get(&decision.route.backend_id) { - routed.headers.extend(headers.clone()); + target_name: &str, + mut request: LibsyRequest, + headers: Map, + streaming: bool, + ) -> Result<(WireProtocol, LlmRequest), LlmClientError> { + let target = + self.config + .targets + .get(target_name) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("libsy selected unknown target {target_name:?}"), + })?; + request.llm_request.stream = streaming; + let mut routed = encode_request( + &self.translation, + target.protocol, + &request.llm_request, + headers, + ) + .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; + let object = routed.content.as_object_mut().ok_or_else(|| { + LlmClientError::RequestEncoding("translated provider request is not an object".into()) + })?; + object.insert("model".into(), Json::String(target.model.clone())); + object.insert("stream".into(), Json::Bool(streaming)); + if let Some(target_headers) = self.target_headers.get(target_name) { + routed.headers.extend(target_headers.clone()); } routed.headers.insert( INTERNAL_DISPATCH_ROUTE_HEADER.into(), - Json::String(binding.protocol.label().into()), + Json::String(target.protocol.label().into()), ); routed.headers.insert( INTERNAL_DISPATCH_URL_HEADER.into(), - Json::String(dispatch_url(&binding.base_url, &binding.endpoint)), + Json::String(dispatch_url(&target.base_url, &target.endpoint)), ); routed.headers.insert( INTERNAL_RETRY_AWARE_HEADER.into(), Json::String("true".into()), ); - Ok(routed) + Ok((target.protocol, routed)) } - fn fallback_request( + fn finish_buffered(&self, inbound: WireProtocol, response: LibsyResponse) -> FlowResult { + match response.llm_response { + LlmResponse::Agg(response) => encode_response(&self.translation, inbound, &response), + LlmResponse::Stream(_) => Err(FlowError::Internal( + "libsy returned a stream for a buffered Relay request".into(), + )), + } + } + + fn finish_stream( &self, inbound: WireProtocol, - request: LlmRequest, - ) -> Result { - let id = self.config.default_targets.target(inbound); - let binding = self - .config - .targets - .get(id) - .ok_or_else(|| format!("unknown fallback target {id:?}"))?; - let decision = RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "relay-fallback".into(), - router: crate::contract::DecisionProvider { - name: "relay-fallback".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "fallback".into(), - target_model: binding.model.clone(), - backend_id: id.to_string(), - target_protocol_profile: binding.protocol.label().into(), - target_endpoint: binding.endpoint.clone(), - }, - baseline_route: None, - confidence: None, - reason_code: Some("relay_trusted_fallback".into()), - reason_summary: None, - metadata: BTreeMap::new(), - extra: BTreeMap::new(), - }; - self.apply_target(inbound, request, &decision) + response: LibsyResponse, + tracker: StreamCloseTracker, + ) -> FlowResult { + let source = response + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format) + .ok_or_else(|| { + FlowError::Internal("libsy streaming response omitted its source format".into()) + })?; + match response.llm_response { + LlmResponse::Stream(response) => Ok(relay_response_stream( + response, + source, + wire_format(inbound), + tracker, + )), + LlmResponse::Agg(_) => Err(FlowError::Internal( + "libsy returned a buffered aggregate for a streaming Relay request".into(), + )), + } } async fn dispatch_fallback_buffered( &self, inbound: WireProtocol, original: LlmRequest, - next: nemo_relay::api::runtime::LlmExecutionNextFn, + next: LlmExecutionNextFn, reason: &str, ) -> FlowResult { self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(response) => Ok(response), - Err(error) => { - emit_terminal_error( - &error, - "fallback_buffered", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } + let request = self.fallback_request(inbound, original, false)?; + next(request).await } async fn dispatch_fallback_stream( &self, inbound: WireProtocol, original: LlmRequest, - next: nemo_relay::api::runtime::LlmStreamExecutionNextFn, + next: LlmStreamExecutionNextFn, reason: &str, ) -> FlowResult { self.emit_fallback(inbound, reason, &original); - let metadata = identity_metadata_from_request(&original); - let request = self - .fallback_request(inbound, original) - .map_err(FlowError::Internal)?; - match next(request).await { - Ok(stream) => Ok(mark_terminal_stream( - stream, - "fallback_stream", - self.config.mode.label(), - metadata.clone(), - )), - Err(error) => { - emit_terminal_error( - &error, - "fallback_stream_setup", - self.config.mode.label(), - metadata, - ); - Err(error) - } - } + let request = self.fallback_request(inbound, original, true)?; + next(request).await } - fn emit_requested(&self, request: &RoutingRequest) { + fn fallback_request( + &self, + inbound: WireProtocol, + original: LlmRequest, + streaming: bool, + ) -> FlowResult { + let target = self.config.default_targets.target(inbound); + let request = self.libsy_request(inbound, &original, streaming)?; + self.apply_target( + target, + request, + dispatch_headers(&original.headers), + streaming, + ) + .map(|(_, request)| request) + .map_err(|error| FlowError::Internal(error.to_string())) + } + + fn emit_requested(&self, request: &LlmRequest, inbound: WireProtocol, attempt: u32) { emit_mark( "switchyard.routing.requested", json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "routing_attempt": request.attempt.routing_attempt, - "profile_id": request.decision_profile.profile_id, - "rollout_mode": self.config.mode.label(), + "algorithm": self.algorithm.name(), + "routing_attempt": attempt, + "inbound_profile": inbound.label(), }), identity_metadata(request), ); } - fn emit_decision( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - observe_only: bool, - latency_ms: u64, - ) { + fn emit_decision(&self, decision: &dyn Decision, attempt: u32, metadata: Json) { emit_mark( "switchyard.routing.decision", json!({ - "decision_id": decision.decision_id, - "profile_id": request.decision_profile.profile_id, - "router": decision.router.name, - "router_version": decision.router.version, + "algorithm": self.algorithm.name(), "routing_attempt": attempt, - "backend_id": decision.route.backend_id, - "selected_tier": decision.route.tier, - "selected_model": decision.route.target_model, - "target_protocol_profile": decision.route.target_protocol_profile, - "target_endpoint": decision.route.target_endpoint, - "confidence": decision.confidence, - "reason_code": decision.reason_code, - "reason_summary": decision.reason_summary, - "router_metadata": decision.metadata, - "latency_ms": latency_ms, - "observe_only": observe_only, - "rollout_mode": self.config.mode.label(), + "semantic_target": decision.selected_model(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + "reasoning": decision.reasoning(), }), - identity_metadata(request), + metadata, ); } - fn emit_retry( - &self, - request: &RoutingRequest, - decision: &RoutingDecision, - attempt: u32, - reason: &str, - ) { + fn emit_call(&self, decision: &dyn Decision, attempt: u32, metadata: Json) { + emit_mark( + "switchyard.routing.call", + json!({ + "algorithm": self.algorithm.name(), + "routing_attempt": attempt, + "semantic_target": decision.selected_model(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + "reasoning": decision.reasoning(), + }), + metadata, + ); + } + + fn emit_retry(&self, attempt: u32, reason: &str, request: &LlmRequest) { emit_mark( "switchyard.routing.retry", - json!({"routing_attempt": attempt, "previous_route": decision.route.backend_id, "retry_reason": reason, "rollout_mode": self.config.mode.label()}), + json!({ + "algorithm": self.algorithm.name(), + "routing_attempt": attempt, + "retry_reason": reason, + }), identity_metadata(request), ); } - fn emit_error(&self, request: Option<&RoutingRequest>, attempt: u32, class: &str, error: &str) { + fn emit_error(&self, attempt: u32, class: &str, error: &str, request: &LlmRequest) { emit_mark( "switchyard.routing.error", - json!({"routing_attempt": attempt, "error_class": class, "error": error, "rollout_mode": self.config.mode.label()}), - request.map(identity_metadata).unwrap_or_else(|| json!({})), + json!({ + "algorithm": self.algorithm.name(), + "routing_attempt": attempt, + "error_class": class, + "error": error, + }), + identity_metadata(request), ); } @@ -1339,190 +932,157 @@ impl SwitchyardRuntime { emit_mark( "switchyard.routing.fallback", json!({ + "algorithm": self.algorithm.name(), "fallback_reason": reason, "fallback_route": self.config.default_targets.target(inbound), "inbound_profile": inbound.label(), - "rollout_mode": self.config.mode.label(), }), - identity_metadata_from_request(request), + identity_metadata(request), ); } } -async fn check_switchyard_health( - client: &reqwest::Client, - health_url: &reqwest::Url, -) -> Result<(), String> { - let response = client - .get(health_url.clone()) - .send() - .await - .map_err(|error| { - format!("Switchyard service is required but health check {health_url} failed: {error}") - })?; - let status = response.status(); - if !status.is_success() { - return Err(format!( - "Switchyard service is required but health check {health_url} returned HTTP {status}" - )); - } - let body = response.json::().await.map_err(|error| { - format!("Switchyard health check {health_url} returned invalid JSON: {error}") - })?; - if body.get("status").and_then(Json::as_str) != Some("ok") { - return Err(format!( - "Switchyard health check {health_url} did not report status=ok" - )); - } - Ok(()) +struct RunFailure { + error: LibsyError, + provider_error: Option, } -fn switchyard_health_url(decision_api_url: &str) -> Result { - let mut url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - url.set_path(SWITCHYARD_HEALTH_PATH); - url.set_query(None); - url.set_fragment(None); - Ok(url) +impl RunFailure { + fn new(error: LibsyError, provider_error: &Arc>>) -> Self { + Self { + error, + provider_error: provider_error.lock().ok().and_then(|error| error.clone()), + } + } + + fn is_retryable(&self) -> bool { + self.provider_error + .as_ref() + .is_some_and(flow_error_is_retryable) + || libsy_error_is_retryable(&self.error) + } } -fn validate_atof_endpoint_name(name: Option<&str>) -> Result, String> { - if let Some(name) = name { - if name.trim().is_empty() { - return Err("atof_endpoint_name must be non-empty when configured".into()); +fn build_algorithm(config: &SwitchyardConfig) -> Result, String> { + let target = |name: &str| { + config + .targets + .contains_key(name) + .then(|| LlmTarget { + semantic_name: name.to_string(), + llm_client: None, + }) + .ok_or_else(|| format!("algorithm target {name:?} is not configured")) + }; + match &config.algorithm { + AlgorithmConfig::Random { seed } => { + let targets = config + .targets + .keys() + .map(|name| target(name)) + .collect::, _>>()?; + let weights = config + .targets + .values() + .map(|target| target.weight) + .collect::>(); + Random::new(LlmTargetSet::new(targets), Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) } - if name != name.trim() { - return Err("atof_endpoint_name must not have leading or trailing whitespace".into()); + AlgorithmConfig::LlmClassifier { + classifier_target, + weak_target, + strong_target, + base_threshold, + min_confidence, + capability_elevated_floor, + session_affinity, + message_hash_fallback, + } => { + let classifier = target(classifier_target)?; + if config.targets[classifier_target].protocol == WireProtocol::AnthropicMessages { + return Err(format!( + "classifier target {classifier_target:?} must use openai_chat or openai_responses because the pinned lossless translation policy cannot encode its structured response format for anthropic_messages" + )); + } + LlmTaskClassifier::new( + classifier, + target(weak_target)?, + target(strong_target)?, + TaskClassifierConfig { + base_threshold: *base_threshold, + min_confidence: *min_confidence, + capability_elevated_floor: *capability_elevated_floor, + session_affinity: *session_affinity, + message_hash_fallback: *message_hash_fallback, + recent_turn_window: None, + }, + ) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) } } - Ok(name) } fn validate_config(config: &SwitchyardConfig) -> Result<(), String> { - validate_scalar_config(config)?; - validate_decision_api_url(&config.decision_api_url)?; - validate_target_bindings(config)?; - validate_default_targets(config) -} - -fn validate_scalar_config(config: &SwitchyardConfig) -> Result<(), String> { - if config.version != 1 { + if config.version != 2 { return Err(format!( - "unsupported Switchyard config version {}", + "unsupported Switchyard config version {}; version 1 used the removed switchyard-server Decision API, use version = 2", config.version )); } - if config.decision_profile_id.trim().is_empty() { - return Err("decision_profile_id must be non-empty".into()); - } - if config.decision_timeout_millis == 0 { - return Err("decision_timeout_millis must be greater than zero".into()); - } if config.max_retries > 10 { return Err("max_retries must not exceed 10".into()); } - if config.recent_message_count == 0 { - return Err("recent_message_count must be greater than zero".into()); - } - let atof_endpoint_name = validate_atof_endpoint_name(config.atof_endpoint_name.as_deref())?; - if config.context_mode == ContextMode::AtofRequired && atof_endpoint_name.is_none() { - return Err("atof_required Switchyard profiles require atof_endpoint_name".into()); - } - Ok(()) -} - -fn validate_decision_api_url(decision_api_url: &str) -> Result<(), String> { - let url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - if !matches!(url.scheme(), "http" | "https") { - return Err("decision_api_url must use http or https".into()); - } - Ok(()) -} - -fn validate_target_bindings(config: &SwitchyardConfig) -> Result<(), String> { if config.targets.is_empty() { return Err("targets must not be empty".into()); } if config.enabled_inbound_profiles.is_empty() { return Err("enabled_inbound_profiles must not be empty".into()); } - let mut exact_bindings = BTreeSet::new(); - for (id, target) in &config.targets { - if id.trim().is_empty() - || target.model.trim().is_empty() - || target.endpoint.trim().is_empty() - { - return Err("target IDs, models, and endpoints must be non-empty".into()); - } - let base_url = reqwest::Url::parse(&target.base_url) - .map_err(|error| format!("target {id:?} base_url is invalid: {error}"))?; - if !matches!(base_url.scheme(), "http" | "https") { - return Err(format!("target {id:?} base_url must use http or https")); + for (name, target) in &config.targets { + if name.trim().is_empty() || target.model.trim().is_empty() { + return Err("target names and models must be non-empty".into()); } if target.endpoint != target.protocol.endpoint() { return Err(format!( - "target {id:?} endpoint must be {:?} for {}", + "target {name:?} endpoint must be {:?} for {}", target.protocol.endpoint(), target.protocol.label() )); } - if !exact_bindings.insert(( - target.model.clone(), - target.protocol, - target.endpoint.clone(), - target.base_url.trim_end_matches('/').to_string(), - )) { + validate_http_url(&target.base_url, name)?; + if !target.weight.is_finite() || target.weight < 0.0 { return Err(format!( - "target {id:?} conflicts with another exact backend binding" + "target {name:?} weight must be finite and nonnegative" )); } } - Ok(()) -} - -fn validate_default_targets(config: &SwitchyardConfig) -> Result<(), String> { for &protocol in &config.enabled_inbound_profiles { - let id = config.default_targets.target(protocol); + let fallback = config.default_targets.target(protocol); let target = config .targets - .get(id) - .ok_or_else(|| format!("default target {id:?} is not configured"))?; + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; if target.protocol != protocol { return Err(format!( - "default target {id:?} must use protocol {}", + "default target {fallback:?} must use protocol {}", protocol.label() )); } } - Ok(()) + build_algorithm(config).map(|_| ()) } -fn resolve_headers( - static_headers: &BTreeMap, - environment_headers: &BTreeMap, -) -> Result { - let mut headers = HeaderMap::new(); - for (name, value) in static_headers { - insert_http_header(&mut headers, name, value)?; +fn validate_http_url(url: &str, target: &str) -> Result<(), String> { + let uri = url + .parse::() + .map_err(|error| format!("target {target:?} base_url is invalid: {error}"))?; + if !matches!(uri.scheme_str(), Some("http" | "https")) || uri.authority().is_none() { + return Err(format!("target {target:?} base_url must use http or https")); } - for (name, variable) in environment_headers { - if static_headers - .keys() - .any(|configured| configured.eq_ignore_ascii_case(name)) - { - return Err(format!( - "header {name:?} cannot appear in both headers and header_env" - )); - } - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - if value.trim().is_empty() { - return Err(format!("environment variable {variable:?} is blank")); - } - insert_http_header(&mut headers, name, &value)?; - } - Ok(headers) + Ok(()) } fn resolve_json_headers( @@ -1531,6 +1091,7 @@ fn resolve_json_headers( ) -> Result, String> { let mut headers = Map::new(); for (name, value) in static_headers { + validate_target_header(name, value)?; headers.insert(name.clone(), Json::String(value.clone())); } for (name, variable) in environment_headers { @@ -1547,41 +1108,103 @@ fn resolve_json_headers( if value.trim().is_empty() { return Err(format!("environment variable {variable:?} is blank")); } + validate_target_header(name, &value)?; headers.insert(name.clone(), Json::String(value)); } Ok(headers) } -fn insert_http_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), String> { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid header name: {error}"))?; - let value = - HeaderValue::from_str(value).map_err(|error| format!("invalid header value: {error}"))?; - headers.insert(name, value); +fn validate_target_header(name: &str, value: &str) -> Result<(), String> { + let normalized = name.to_ascii_lowercase(); + if matches!( + normalized.as_str(), + INTERNAL_DISPATCH_URL_HEADER | INTERNAL_DISPATCH_ROUTE_HEADER | INTERNAL_RETRY_AWARE_HEADER + ) { + return Err(format!( + "target header {name:?} is reserved for Relay dispatch" + )); + } + HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; Ok(()) } -fn protocol_from_label(label: &str) -> FlowResult { - match label { - "openai_chat" | "openai_chat_completions" | "openai_chat_completions.v1" => { - Ok(WireProtocol::OpenaiChat) +fn response_metadata(source: Option<&Metadata>, protocol: WireProtocol) -> Metadata { + let mut metadata = source.cloned().unwrap_or_default(); + metadata.wire_format = Some(wire_format(protocol)); + metadata +} + +fn context_values(metadata: Option<&Metadata>, attempt: u32) -> BTreeMap { + let mut values = BTreeMap::from([ + ("relay.routing_attempt".into(), attempt.to_string()), + ("relay.scope_id".into(), task_scope_top().uuid.to_string()), + ]); + if let Some(metadata) = metadata { + for (key, value) in [ + ("relay.session_id", metadata.session_id.as_deref()), + ("relay.agent_id", metadata.agent_id.as_deref()), + ("relay.turn_id", metadata.turn_id.as_deref()), + ("relay.correlation_id", metadata.correlation_id.as_deref()), + ] { + if let Some(value) = value { + values.insert(key.into(), value.into()); + } } - "openai_responses" | "openai_responses.v1" => Ok(WireProtocol::OpenaiResponses), - "anthropic_messages" | "anthropic_messages.v1" => Ok(WireProtocol::AnthropicMessages), - value => Err(FlowError::InvalidArgument(format!( - "unsupported Switchyard target protocol {value:?}" - ))), } + values +} + +fn string_headers(headers: &Map) -> BTreeMap { + headers + .iter() + .filter_map(|(name, value)| value.as_str().map(|value| (name.clone(), value.into()))) + .collect() +} + +fn dispatch_headers(headers: &Map) -> Map { + headers + .iter() + .filter(|(name, _)| { + let normalized = name.to_ascii_lowercase(); + !matches!( + normalized.as_str(), + "authorization" | "proxy-authorization" | "x-api-key" | "api-key" + ) && !normalized.starts_with("x-nemo-relay-") + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +fn relay_metadata(headers: &BTreeMap) -> Metadata { + let mut metadata = Metadata::from_headers(headers); + if metadata.turn_id.is_none() { + metadata.turn_id = header_value(headers, RELAY_TURN_ID_HEADER).map(ToOwned::to_owned); + } + if metadata.correlation_id.is_none() { + metadata.correlation_id = + header_value(headers, RELAY_REQUEST_ID_HEADER).map(ToOwned::to_owned); + } + metadata +} + +fn libsy_identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "request_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) } -fn header(request: &LlmRequest, name: &str) -> Option { - request - .headers - .get(name) - .and_then(Json::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) +fn header_value<'a>(headers: &'a BTreeMap, name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + .filter(|value| !value.trim().is_empty()) } fn dispatch_url(base_url: &str, endpoint: &str) -> String { @@ -1594,26 +1217,40 @@ fn dispatch_url(base_url: &str, endpoint: &str) -> String { format!("{base}{endpoint}") } -fn identity_metadata(request: &RoutingRequest) -> Json { - json!({ - "session_id": request.identity.session_id, - "request_id": request.identity.request_id, - "turn_id": request.identity.turn_id, - "owner_id": request.identity.owner_id, - }) +fn remember_provider_error(slot: &Arc>>, error: &FlowError) { + if let Ok(mut stored) = slot.lock() { + *stored = Some(error.clone()); + } } -fn identity_metadata_from_request(request: &LlmRequest) -> Json { - json!({ - "session_id": header(request, "x-nemo-relay-session-id"), - "request_id": header(request, "x-nemo-relay-request-id"), - "turn_id": header(request, "x-nemo-relay-turn-id"), - "owner_id": header(request, "x-nemo-relay-owner-id"), - }) +fn flow_error_is_retryable(error: &FlowError) -> bool { + matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) } -fn error_is_retryable(error: &FlowError) -> bool { - matches!(error, FlowError::Upstream(failure) if failure.is_retryable()) +fn libsy_error_is_retryable(error: &LibsyError) -> bool { + match error { + LibsyError::ClientCall { source, .. } => match source { + LlmClientError::Transport { .. } + | LlmClientError::Timeout { .. } + | LlmClientError::ContextWindowExceeded { .. } => true, + LlmClientError::UpstreamHttp { status, .. } => { + matches!(*status, 408 | 409 | 425 | 429) || *status >= 500 + } + _ => false, + }, + _ => false, + } +} + +fn identity_metadata(request: &LlmRequest) -> Json { + let headers = string_headers(&request.headers); + let metadata = relay_metadata(&headers); + json!({ + "session_id": metadata.session_id, + "agent_id": metadata.agent_id, + "turn_id": metadata.turn_id, + "request_id": metadata.correlation_id, + }) } fn emit_mark(name: &str, data: Json, metadata: Json) { @@ -1624,7 +1261,7 @@ fn emit_mark(name: &str, data: Json, metadata: Json) { .data_schema( DataSchema::builder() .name(ROUTING_MARK_SCHEMA) - .version("1") + .version("2") .build(), ) .metadata(metadata) @@ -1636,184 +1273,6 @@ fn emit_mark(name: &str, data: Json, metadata: Json) { } } -fn emit_terminal_error(error: &FlowError, phase: &str, rollout_mode: &str, metadata: Json) { - emit_mark( - "switchyard.routing.terminal_error", - json!({"error_class": provider_error_class(error), "error": provider_error_summary(error), "phase": phase, "rollout_mode": rollout_mode}), - metadata, - ); -} - -fn provider_error_class(error: &FlowError) -> &'static str { - match error { - FlowError::Upstream(failure) => match failure.class { - nemo_relay::error::UpstreamFailureClass::Connection => "connection", - nemo_relay::error::UpstreamFailureClass::Timeout => "timeout", - nemo_relay::error::UpstreamFailureClass::RetryableStatus => "retryable_status", - nemo_relay::error::UpstreamFailureClass::ContextWindow => "context_window", - nemo_relay::error::UpstreamFailureClass::ModelUnavailable => "model_unavailable", - nemo_relay::error::UpstreamFailureClass::Authentication => "authentication", - nemo_relay::error::UpstreamFailureClass::InvalidRequest => "invalid_request", - nemo_relay::error::UpstreamFailureClass::Other => "other", - }, - _ => "relay", - } -} - -fn provider_error_summary(error: &FlowError) -> String { - match error { - FlowError::Upstream(failure) => match failure.status { - Some(status) => format!("{}:http_{status}", provider_error_class(error)), - None => provider_error_class(error).to_string(), - }, - _ => error.to_string(), - } -} - -struct PrefixedStream { - first: Option>, - upstream: LlmJsonStream, -} - -impl futures_util::Stream for PrefixedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(first) = self.first.take() { - Poll::Ready(Some(first)) - } else { - Pin::new(&mut self.upstream).poll_next(cx) - } - } -} - -impl LlmStreamInner for PrefixedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.first = None; - Box::pin(async move { this.upstream.close().await }) - } -} - -struct TerminalMarkedStream { - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, - finished: bool, -} - -impl futures_util::Stream for TerminalMarkedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if self.finished { - return Poll::Ready(None); - } - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => Poll::Ready(Some(Ok(chunk))), - Poll::Ready(Some(Err(error))) => { - self.finished = true; - emit_terminal_error(&error, self.phase, self.rollout_mode, self.metadata.clone()); - Poll::Ready(Some(Err(error))) - } - Poll::Ready(None) => { - self.finished = true; - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } -} - -impl LlmStreamInner for TerminalMarkedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - Box::pin(async move { self.get_mut().upstream.close().await }) - } -} - -struct TranslatedStream { - upstream: LlmJsonStream, - transcoder: StreamTranscoder, - buffered: VecDeque>, - upstream_finished: bool, -} - -impl futures_util::Stream for TranslatedStream { - type Item = FlowResult; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - loop { - if let Some(chunk) = self.buffered.pop_front() { - return Poll::Ready(Some(chunk)); - } - if self.upstream_finished { - return Poll::Ready(None); - } - - match Pin::new(&mut self.upstream).poll_next(cx) { - Poll::Ready(Some(Ok(chunk))) => match self.transcoder.transcode(&chunk) { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - }, - Poll::Ready(Some(Err(error))) => { - self.upstream_finished = true; - return Poll::Ready(Some(Err(error))); - } - Poll::Ready(None) => { - self.upstream_finished = true; - match self.transcoder.finish() { - Ok(chunks) => self.buffered.extend(chunks.into_iter().map(Ok)), - Err(error) => return Poll::Ready(Some(Err(error))), - } - } - Poll::Pending => return Poll::Pending, - } - } - } -} - -impl LlmStreamInner for TranslatedStream { - fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { - let this = self.get_mut(); - this.buffered.clear(); - this.upstream_finished = true; - Box::pin(async move { this.upstream.close().await }) - } -} - -fn mark_terminal_stream( - upstream: LlmJsonStream, - phase: &'static str, - rollout_mode: &'static str, - metadata: Json, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TerminalMarkedStream { - upstream, - phase, - rollout_mode, - metadata, - finished: false, - }) -} - -fn translated_stream( - source: WireProtocol, - target: WireProtocol, - effective_model: String, - upstream: LlmJsonStream, -) -> LlmJsonStream { - LlmJsonStream::from_closeable(TranslatedStream { - upstream, - transcoder: StreamTranscoder::new(source, target, effective_model), - buffered: VecDeque::new(), - upstream_finished: false, - }) -} - #[cfg(test)] #[path = "../tests/unit/component_tests.rs"] mod tests; diff --git a/crates/switchyard/src/contract.rs b/crates/switchyard/src/contract.rs deleted file mode 100644 index 675234ca1..000000000 --- a/crates/switchyard/src/contract.rs +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! JSON compatibility types for the Switchyard Decision API. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// Routing request schema supported by this plugin. -pub const ROUTING_REQUEST_SCHEMA_VERSION: &str = "switchyard.routing_request.v1"; -/// Routing decision schema supported by this plugin. -pub const ROUTING_DECISION_SCHEMA_VERSION: &str = "switchyard.routing_decision.v1"; - -/// Request-time materialization supplied to Switchyard. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "snake_case")] -pub enum RequestMaterialization { - /// Identity, protocol, summary, and attempt only. - None, - /// Baseline summary without current request material. - SummaryOnly, - /// Latest user prompt. - LatestUserPrompt, - /// Bounded recent message window. - RecentMessageWindow, - /// Relay-normalized request plus its provider body. - AnnotatedRequest, - /// Complete provider request body. - FullBody, -} - -/// Switchyard profile selection. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProfile { - /// Profile ID loaded by Switchyard. - pub profile_id: String, - /// Request materialization mode. - pub request_materialization: RequestMaterialization, -} - -/// Normalized Relay identity. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestIdentity { - /// Stable session identifier. - pub session_id: String, - /// Per-request identifier. - pub request_id: String, - /// Optional turn identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub turn_id: Option, - /// Optional parent scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_scope_id: Option, - /// Optional root scope identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub root_scope_id: Option, - /// Harness name. - pub harness: String, - /// Request source. - pub source: String, - /// Optional resolved work owner. - #[serde(skip_serializing_if = "Option::is_none")] - pub owner_id: Option, - /// Native, explicit, or synthetic identity quality. - pub quality: String, -} - -/// Inbound protocol context. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RequestProtocol { - /// Inbound protocol profile. - pub inbound_profile: String, - /// Inbound endpoint. - pub inbound_endpoint: String, - /// Response profile expected by the harness. - pub desired_response_profile: String, -} - -/// Cheap provider-request summary. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct RequestSummary { - /// Client-requested model. - #[serde(skip_serializing_if = "Option::is_none")] - pub client_requested_model: Option, - /// Optional prompt-token estimate. - #[serde(skip_serializing_if = "Option::is_none")] - pub prompt_token_estimate: Option, - /// Number of tools in the request. - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_count_in_payload: Option, - /// Whether a system prompt is present. - #[serde(skip_serializing_if = "Option::is_none")] - pub has_system_prompt: Option, -} - -/// Routing-attempt context. Additive fields are ignored by older Switchyard servers. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionAttempt { - /// One-indexed routing attempt. - pub routing_attempt: u32, - /// Maximum Decision API attempts. - pub max_routing_attempts: u32, - /// Previously selected backend. - #[serde(skip_serializing_if = "Option::is_none")] - pub previous_route: Option, - /// Reason another decision is requested. - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_reason: Option, -} - -/// Canonical routing request sent by Relay. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingRequest { - /// Schema identifier. - pub schema_version: String, - /// Switchyard profile selection. - pub decision_profile: DecisionProfile, - /// Normalized identity. - pub identity: RequestIdentity, - /// Inbound protocol. - pub protocol: RequestProtocol, - /// Cheap request summary. - pub request_summary: RequestSummary, - /// Optional request materialization. - #[serde(skip_serializing_if = "Option::is_none")] - pub current_request: Option, - /// Attempt metadata. - pub attempt: DecisionAttempt, -} - -/// Router metadata returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct DecisionProvider { - /// Router name. - pub name: String, - /// Router version. - pub version: String, -} - -/// Selected target returned by Switchyard. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingTarget { - /// Tier label. - pub tier: String, - /// Selected model. - pub target_model: String, - /// Backend binding ID. - pub backend_id: String, - /// Target protocol profile. - pub target_protocol_profile: String, - /// Target endpoint. - pub target_endpoint: String, -} - -/// Canonical Switchyard routing decision. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct RoutingDecision { - /// Schema identifier. - pub schema_version: String, - /// Decision identifier. - pub decision_id: String, - /// Router metadata. - pub router: DecisionProvider, - /// Selected target. - pub route: RoutingTarget, - /// Explicit counterfactual route supplied by routers that define one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub baseline_route: Option, - /// Optional confidence. - #[serde(default)] - pub confidence: Option, - /// Optional reason code. - #[serde(default)] - pub reason_code: Option, - /// Optional reason summary. - #[serde(default)] - pub reason_summary: Option, - /// Additive router metadata. - #[serde(default)] - pub metadata: BTreeMap, - /// Unknown additive response fields. - #[serde(flatten)] - pub extra: BTreeMap, -} diff --git a/crates/switchyard/src/lib.rs b/crates/switchyard/src/lib.rs index 238d4337f..a0168e4e9 100644 --- a/crates/switchyard/src/lib.rs +++ b/crates/switchyard/src/lib.rs @@ -3,15 +3,13 @@ #![deny(rustdoc::broken_intra_doc_links, rustdoc::private_intra_doc_links)] -//! First-party Switchyard Decision API routing plugin for NeMo Relay. +//! First-party in-process Switchyard libsy routing plugin for NeMo Relay. pub mod component; -pub mod contract; mod stream_translation; mod translation; pub use component::{ - ContextMode, ProtocolDefaults, RoutingMode, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, - TargetBinding, WireProtocol, deregister_switchyard_component, register_switchyard_component, - validate_switchyard_atof_configuration, + AlgorithmConfig, ProtocolDefaults, SWITCHYARD_PLUGIN_KIND, SwitchyardConfig, TargetBinding, + WireProtocol, deregister_switchyard_component, register_switchyard_component, }; diff --git a/crates/switchyard/src/stream_translation.rs b/crates/switchyard/src/stream_translation.rs index 808834a2b..6e5e846a9 100644 --- a/crates/switchyard/src/stream_translation.rs +++ b/crates/switchyard/src/stream_translation.rs @@ -1,94 +1,386 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Relay stream adapter for Switchyard's provider translation engine. +//! Relay stream adapters for Switchyard provider events. -use nemo_relay::error::{FlowError, Result}; +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; + +use futures_util::Stream; +use nemo_relay::api::runtime::{LlmJsonStream, LlmStreamInner}; +use nemo_relay::error::{FlowError, Result as FlowResult, UpstreamFailure, UpstreamFailureClass}; use serde_json::Value as Json; +use switchyard_protocol::{LlmClientError, LlmResponseChunk, LlmResponseStream, WireFormat}; use switchyard_translation::{StreamTranslationState, TranslationEngine}; +use tokio::sync::oneshot; + +/// Completion handles for provider streams owned by one libsy run. +#[derive(Clone, Default)] +pub(crate) struct StreamCloseTracker { + receivers: Arc>>>>, +} + +impl StreamCloseTracker { + fn track(&self, receiver: oneshot::Receiver>) { + if let Ok(mut receivers) = self.receivers.lock() { + receivers.push(receiver); + } + } -use crate::component::WireProtocol; -use crate::translation::wire_format; + async fn close_all(&self) -> FlowResult<()> { + let receivers = self + .receivers + .lock() + .map_err(|error| FlowError::Internal(error.to_string()))? + .drain(..) + .collect::>(); + for receiver in receivers { + match receiver.await { + Ok(result) => result?, + Err(_) => { + return Err(FlowError::Internal( + "Switchyard provider stream close task was dropped".into(), + )); + } + } + } + Ok(()) + } +} -pub(crate) struct StreamTranscoder { - engine: TranslationEngine, - source: WireProtocol, - target: WireProtocol, +/// Converts one Relay provider stream into libsy's neutral stream contract. +pub(crate) fn provider_response_stream( + upstream: LlmJsonStream, + source: WireFormat, + first: Json, + tracker: StreamCloseTracker, + model: String, +) -> std::result::Result { + let translation = TranslationEngine::default(); + let mut state = StreamTranslationState::new(source, source); + let first = decode_provider_event(&translation, &mut state, source, first)?; + let (closed_tx, closed_rx) = oneshot::channel(); + tracker.track(closed_rx); + Ok(Box::pin(RelayProviderStream { + upstream: Some(upstream), + first: Some(first), + source, + translation, + state, + closed_tx: Some(closed_tx), + model, + })) +} + +struct RelayProviderStream { + upstream: Option, + first: Option, + source: WireFormat, + translation: TranslationEngine, state: StreamTranslationState, + closed_tx: Option>>, + model: String, } -impl StreamTranscoder { - pub(crate) fn new( - source: WireProtocol, - target: WireProtocol, - effective_model: impl Into, - ) -> Self { - let effective_model = effective_model.into(); - let mut state = StreamTranslationState::new(wire_format(source), wire_format(target)); - state.model = Some(effective_model.clone()); - state.target_model = Some(effective_model); - Self { - engine: TranslationEngine::default(), - source, - target, - state, +impl RelayProviderStream { + fn decode(&mut self, raw: Json) -> std::result::Result { + decode_provider_event(&self.translation, &mut self.state, self.source, raw) + } +} + +fn decode_provider_event( + translation: &TranslationEngine, + state: &mut StreamTranslationState, + source: WireFormat, + raw: Json, +) -> std::result::Result { + let chunk = translation + .decode_stream_event(state, source, &raw) + .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?; + ensure_provider_event_succeeded(&chunk)?; + Ok(chunk) +} + +fn ensure_provider_event_succeeded( + chunk: &LlmResponseChunk, +) -> std::result::Result<(), LlmClientError> { + match chunk { + LlmResponseChunk::ProviderEvent { normalized, .. } => { + for chunk in normalized { + ensure_provider_event_succeeded(chunk)?; + } + Ok(()) + } + LlmResponseChunk::DecodeError { message } => { + Err(LlmClientError::ResponseTranslation(message.clone())) } + LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp { + status: 502, + body: message.clone(), + }), + _ => Ok(()), } +} - pub(crate) fn transcode(&mut self, chunk: &Json) -> Result> { - if unsupported_stream_chunk(self.source, chunk) { - return Err(FlowError::InvalidArgument( - "provider-specific streaming extension cannot be translated safely".into(), - )); +impl Stream for RelayProviderStream { + type Item = std::result::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + if let Some(first) = this.first.take() { + return Poll::Ready(Some(Ok(first))); + } + let Some(upstream) = this.upstream.as_mut() else { + return Poll::Ready(None); + }; + match Pin::new(upstream).poll_next(cx) { + Poll::Ready(Some(Ok(raw))) => Poll::Ready(Some(this.decode(raw))), + Poll::Ready(Some(Err(error))) => { + Poll::Ready(Some(Err(flow_to_client_error(error, &this.model)))) + } + Poll::Ready(None) => { + this.upstream.take(); + if let Some(closed_tx) = this.closed_tx.take() { + let _ = closed_tx.send(Ok(())); + } + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, } - self.engine - .translate_event( - &mut self.state, - wire_format(self.source), - wire_format(self.target), - chunk, - ) - .map_err(|error| { - FlowError::InvalidArgument(format!("Switchyard stream translation failed: {error}")) - }) } +} - pub(crate) fn finish(&mut self) -> Result> { - self.engine - .finish_stream(&mut self.state, wire_format(self.target)) - .map_err(|error| { - FlowError::InvalidArgument(format!( - "Switchyard stream finalization failed: {error}" - )) - }) +impl Drop for RelayProviderStream { + fn drop(&mut self) { + let Some(mut upstream) = self.upstream.take() else { + return; + }; + let closed_tx = self.closed_tx.take(); + tokio::spawn(async move { + let result = upstream.close().await; + if let Some(closed_tx) = closed_tx { + let _ = closed_tx.send(result); + } + }); } } -fn unsupported_stream_chunk(source: WireProtocol, chunk: &Json) -> bool { - match source { - WireProtocol::OpenaiChat => { - chunk["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["delta"].get("audio").is_some() - || choice["delta"].get("reasoning_content").is_some() - }) +/// Converts libsy's final response stream back into Relay provider events. +pub(crate) fn relay_response_stream( + upstream: LlmResponseStream, + source: WireFormat, + target: WireFormat, + tracker: StreamCloseTracker, +) -> LlmJsonStream { + LlmJsonStream::from_closeable(LibsyResponseStream { + upstream: Some(upstream), + source, + target, + translation: TranslationEngine::default(), + state: StreamTranslationState::new(source, target), + buffered: VecDeque::new(), + finished: false, + tracker, + }) +} + +struct LibsyResponseStream { + upstream: Option, + source: WireFormat, + target: WireFormat, + translation: TranslationEngine, + state: StreamTranslationState, + buffered: VecDeque>, + finished: bool, + tracker: StreamCloseTracker, +} + +impl Stream for LibsyResponseStream { + type Item = FlowResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + loop { + if let Some(item) = this.buffered.pop_front() { + return Poll::Ready(Some(item)); + } + if this.finished { + return Poll::Ready(None); + } + let Some(upstream) = this.upstream.as_mut() else { + this.finished = true; + return Poll::Ready(None); + }; + match upstream.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(event))) => { + match this + .translation + .encode_stream_event(&mut this.state, this.target, event) + { + Ok(events) => { + this.buffered.extend(events.into_iter().map(Ok)); + } + Err(error) => { + this.finished = true; + return Poll::Ready(Some(Err(FlowError::InvalidArgument(format!( + "Switchyard stream translation failed: {error}" + ))))); + } + } + } + Poll::Ready(Some(Err(error))) => { + this.finished = true; + return Poll::Ready(Some(Err(client_to_flow_error(error)))); + } + Poll::Ready(None) => { + this.upstream.take(); + this.finished = true; + if this.source == this.target { + return Poll::Ready(None); + } + match this.translation.finish_stream(&mut this.state, this.target) { + Ok(events) => { + this.buffered.extend(events.into_iter().map(Ok)); + } + Err(error) => { + return Poll::Ready(Some(Err(FlowError::InvalidArgument(format!( + "Switchyard stream finalization failed: {error}" + ))))); + } + } + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +impl LlmStreamInner for LibsyResponseStream { + fn close(self: Pin<&mut Self>) -> Pin> + Send + '_>> { + let this = self.get_mut(); + this.upstream.take(); + this.buffered.clear(); + this.finished = true; + Box::pin(async move { this.tracker.close_all().await }) + } +} + +pub(crate) fn flow_to_client_error(error: FlowError, model: &str) -> LlmClientError { + match error { + FlowError::Upstream(failure) => match failure.class { + UpstreamFailureClass::Timeout => LlmClientError::Timeout { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + failure.body, + )), + }, + UpstreamFailureClass::Connection => LlmClientError::Transport { + source: Box::new(std::io::Error::other(failure.body)), + }, + UpstreamFailureClass::ContextWindow => LlmClientError::ContextWindowExceeded { + model: model.into(), + message: failure.body, + }, + UpstreamFailureClass::ModelUnavailable => LlmClientError::General(failure.body), + UpstreamFailureClass::InvalidRequest => LlmClientError::InvalidRequest { + message: failure.body, + }, + UpstreamFailureClass::Authentication | UpstreamFailureClass::Other + if failure.status.is_none() => + { + LlmClientError::General(failure.body) + } + _ => LlmClientError::UpstreamHttp { + status: failure.status.unwrap_or(500), + body: failure.body, + }, + }, + FlowError::InvalidArgument(message) => LlmClientError::InvalidRequest { message }, + other => LlmClientError::General(other.to_string()), + } +} + +fn client_to_flow_error(error: LlmClientError) -> FlowError { + match error { + LlmClientError::Transport { source } => { + provider_failure(None, source.to_string(), UpstreamFailureClass::Connection) } - WireProtocol::OpenaiResponses => { - chunk.get("type").and_then(Json::as_str) == Some("response.output_item.added") - && matches!( - chunk["item"].get("type").and_then(Json::as_str), - Some("reasoning" | "computer_call" | "web_search_call") - ) + LlmClientError::Timeout { source } => { + provider_failure(None, source.to_string(), UpstreamFailureClass::Timeout) } - WireProtocol::AnthropicMessages => { - chunk.get("type").and_then(Json::as_str) == Some("content_block_start") - && !matches!( - chunk["content_block"].get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) + LlmClientError::ContextWindowExceeded { message, .. } => { + provider_failure(None, message, UpstreamFailureClass::ContextWindow) } + LlmClientError::UpstreamHttp { status, body } => { + let class = if status == 401 || status == 403 { + UpstreamFailureClass::Authentication + } else if matches!(status, 408 | 409 | 425 | 429) || status >= 500 { + UpstreamFailureClass::RetryableStatus + } else if (400..500).contains(&status) { + UpstreamFailureClass::InvalidRequest + } else { + UpstreamFailureClass::Other + }; + provider_failure(Some(status), body, class) + } + LlmClientError::InvalidRequest { message } + | LlmClientError::RequestTranslation(message) + | LlmClientError::RequestEncoding(message) + | LlmClientError::ResponseTranslation(message) + | LlmClientError::Configuration { message } => FlowError::InvalidArgument(message), + other => FlowError::Internal(format!("Switchyard model call failed: {other}")), + } +} + +fn provider_failure(status: Option, body: String, class: UpstreamFailureClass) -> FlowError { + FlowError::Upstream(UpstreamFailure { + status, + body, + headers: Default::default(), + class, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::StreamExt; + use serde_json::json; + + #[tokio::test] + async fn preserved_translation_replays_unknown_same_protocol_stream_fields() { + let raw = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "provider/model", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "hello"}, + "finish_reason": null + }] + }); + let upstream = LlmJsonStream::new(futures_util::stream::iter([Ok(raw.clone())])); + let tracker = StreamCloseTracker::default(); + let neutral = provider_response_stream( + upstream, + WireFormat::OpenAiChat, + raw.clone(), + tracker.clone(), + "target".into(), + ) + .unwrap(); + let mut replay = relay_response_stream( + neutral, + WireFormat::OpenAiChat, + WireFormat::OpenAiChat, + tracker, + ); + let output = replay.next().await.and_then(Result::ok); + + assert_eq!(output, Some(raw)); } } diff --git a/crates/switchyard/src/translation.rs b/crates/switchyard/src/translation.rs index d1640226c..689e9890f 100644 --- a/crates/switchyard/src/translation.rs +++ b/crates/switchyard/src/translation.rs @@ -1,13 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +//! Relay adapters for Switchyard's buffered translation engine. + use nemo_relay::api::llm::LlmRequest; use nemo_relay::error::{FlowError, Result}; use serde_json::{Map, Value as Json}; +use switchyard_protocol::{AggLlmResponse, LlmRequest as ProtocolRequest}; use switchyard_translation::{ - ContentBlock, DeterministicIdPolicy, ImageSource, LlmRequest as TranslationRequest, - LossyConversionPolicy, PreservationPolicy, Role, TargetCapabilities, TranslationDiagnostic, - TranslationEngine, TranslationPolicy, UnknownFieldPolicy, WireFormat, + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, WireFormat, }; use crate::component::WireProtocol; @@ -20,7 +23,7 @@ pub(crate) fn decode_request( engine: &TranslationEngine, protocol: WireProtocol, request: &LlmRequest, -) -> Result { +) -> Result { let output = engine .decode_request( wire_format(protocol), @@ -28,106 +31,51 @@ pub(crate) fn decode_request( &translation_policy(), ) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(output.request) } pub(crate) fn encode_request( engine: &TranslationEngine, protocol: WireProtocol, - request: &TranslationRequest, + request: &ProtocolRequest, headers: Map, ) -> Result { let output = engine - .encode_request(wire_format(protocol), request, &translation_policy()) + .encode_request( + wire_format(protocol), + request, + &request_translation_policy(protocol), + ) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(LlmRequest { headers, content: output.body, }) } -pub(crate) fn validate_portable_request( +pub(crate) fn decode_response( engine: &TranslationEngine, protocol: WireProtocol, - request: &LlmRequest, -) -> Result<()> { - const RESTRICTED_KEYS: &[&str] = &[ - "cache_control", - "audio", - "thinking", - "computer_use", - "server_tool_use", - ]; - if protocol == WireProtocol::AnthropicMessages - && contains_invalid_anthropic_image_source(&request.content) - { - return Err(FlowError::InvalidArgument( - "request uses an unsupported or malformed Anthropic image source".into(), - )); - } - if contains_any_key_recursive(&request.content, RESTRICTED_KEYS) { - return Err(FlowError::InvalidArgument( - "request uses a provider-specific extension that requires same-protocol fail-open" - .into(), - )); - } - let decoded = decode_request(engine, protocol, request)?; - if decoded.reasoning.effort.is_some() - || decoded.reasoning.raw.is_some() - || decoded - .extensions - .fields - .iter() - .any(|(key, value)| key != "stream_options" || !portable_stream_options(value)) - || request_contains_unsupported_content(&decoded) - { - return Err(FlowError::InvalidArgument( - "request uses provider-specific fields that cannot be translated safely".into(), - )); - } - Ok(()) -} - -pub(crate) fn latest_user_prompt(request: &TranslationRequest) -> Option { - request - .messages - .iter() - .rev() - .find(|message| message.role == Role::User) - .and_then(|message| message.text_content("\n")) -} - -pub(crate) fn recent_message_window( - request: &TranslationRequest, - count: usize, -) -> TranslationRequest { - let mut window = request.clone(); - let split = window.messages.len().saturating_sub(count); - window.messages = window.messages.split_off(split); - window + response: &Json, +) -> Result { + let output = engine + .decode_response(wire_format(protocol), response, &translation_policy()) + .map_err(translation_error)?; + ensure_safe_diagnostics(&output.diagnostics)?; + Ok(output.response) } -pub(crate) fn translate_response( +pub(crate) fn encode_response( engine: &TranslationEngine, - source: WireProtocol, - target: WireProtocol, - response: &Json, + protocol: WireProtocol, + response: &AggLlmResponse, ) -> Result { - if source == target { - return Ok(response.clone()); - } - ensure_portable_response(source, response)?; let output = engine - .translate_response( - wire_format(source), - wire_format(target), - response, - &translation_policy(), - ) + .encode_response(wire_format(protocol), response, &translation_policy()) .map_err(translation_error)?; - ensure_no_diagnostics(&output.diagnostics)?; + ensure_safe_diagnostics(&output.diagnostics)?; Ok(output.body) } @@ -141,176 +89,241 @@ pub(crate) const fn wire_format(protocol: WireProtocol) -> WireFormat { fn translation_policy() -> TranslationPolicy { TranslationPolicy { - unknown_field_policy: UnknownFieldPolicy::Reject, + unknown_field_policy: UnknownFieldPolicy::Preserve, lossy_conversion_policy: LossyConversionPolicy::Reject, deterministic_ids: DeterministicIdPolicy::GenerateStable { prefix: "relay".into(), }, - preservation: PreservationPolicy::Disabled, + preservation: PreservationPolicy::InMemory, target_capabilities: TargetCapabilities::default(), } } +fn request_translation_policy(protocol: WireProtocol) -> TranslationPolicy { + let mut policy = translation_policy(); + if protocol == WireProtocol::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + fn translation_error(error: switchyard_translation::TranslationError) -> FlowError { FlowError::InvalidArgument(format!("Switchyard translation failed: {error}")) } -fn ensure_no_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { - if diagnostics.is_empty() { +fn ensure_safe_diagnostics(diagnostics: &[TranslationDiagnostic]) -> Result<()> { + let lossy = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if lossy.is_empty() { Ok(()) } else { Err(FlowError::InvalidArgument(format!( - "Switchyard translation was not lossless: {diagnostics:?}" + "Switchyard translation was not lossless: {lossy:?}" ))) } } -fn portable_stream_options(value: &Json) -> bool { - let Some(options) = value.as_object() else { - return false; - }; - options.len() == 1 && options.get("include_usage").is_some_and(Json::is_boolean) -} +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; -fn request_contains_unsupported_content(request: &TranslationRequest) -> bool { - request - .instructions - .iter() - .flat_map(|instruction| instruction.content.iter()) - .chain( - request - .messages - .iter() - .flat_map(|message| message.content.iter()), - ) - .any(unsupported_content_block) -} + const PROTOCOLS: [WireProtocol; 3] = [ + WireProtocol::OpenaiChat, + WireProtocol::OpenaiResponses, + WireProtocol::AnthropicMessages, + ]; -fn unsupported_content_block(block: &ContentBlock) -> bool { - match block { - ContentBlock::Text { .. } | ContentBlock::Refusal { .. } | ContentBlock::ToolCall(_) => { - false - } - ContentBlock::Image { source } => invalid_image_source(source), - ContentBlock::ToolResult(result) => result.content.iter().any(unsupported_content_block), - ContentBlock::Reasoning { .. } - | ContentBlock::Audio { .. } - | ContentBlock::Video { .. } - | ContentBlock::File { .. } - | ContentBlock::Unknown { .. } => true, + #[test] + fn same_protocol_buffered_response_preserves_provider_extensions() { + let engine = translation_engine(); + let original = json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": "provider/model", + "system_fingerprint": "fp_exact", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop" + }] + }); + + let decoded = decode_response(&engine, WireProtocol::OpenaiChat, &original); + assert!(decoded.is_ok()); + let encoded = decoded + .and_then(|response| encode_response(&engine, WireProtocol::OpenaiChat, &response)); + assert_eq!(encoded.ok(), Some(original)); } -} -fn invalid_image_source(source: &ImageSource) -> bool { - match source { - ImageSource::Url { url, .. } => { - url.starts_with("data:") && base64_data_uri_parts(url).is_none() - } - ImageSource::Base64 { media_type, data } => { - media_type.as_deref().is_none_or(str::is_empty) || data.is_empty() - } - ImageSource::Raw(_) => true, + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let engine = translation_engine(); + let original = LlmRequest { + headers: Map::new(), + content: json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + + let decoded = decode_request(&engine, WireProtocol::OpenaiChat, &original); + assert!(decoded.is_ok()); + let encoded = decoded.and_then(|request| { + encode_request(&engine, WireProtocol::OpenaiChat, &request, Map::new()) + }); + assert_eq!( + encoded.ok().map(|request| request.content), + Some(original.content) + ); } -} -fn base64_data_uri_parts(url: &str) -> Option<(&str, &str)> { - let (metadata, data) = url.strip_prefix("data:")?.split_once(',')?; - let media_type = metadata.strip_suffix(";base64")?; - (!media_type.is_empty() && !data.is_empty()).then_some((media_type, data)) -} + #[test] + fn all_same_protocol_requests_and_responses_preserve_unknown_fields() { + let engine = translation_engine(); + for protocol in PROTOCOLS { + let request = request_fixture(protocol, true); + let decoded = decode_request( + &engine, + protocol, + &LlmRequest { + headers: Map::new(), + content: request.clone(), + }, + ) + .expect("request should decode"); + let encoded = encode_request(&engine, protocol, &decoded, Map::new()) + .expect("request should encode"); + assert_eq!( + encoded.content, request, + "request mismatch for {protocol:?}" + ); -fn contains_invalid_anthropic_image_source(value: &Json) -> bool { - match value { - Json::Object(object) => { - if object.get("type").and_then(Json::as_str) == Some("image") { - let Some(source) = object.get("source").and_then(Json::as_object) else { - return true; - }; - match source.get("type").and_then(Json::as_str) { - Some("url") => source - .get("url") - .and_then(Json::as_str) - .is_none_or(str::is_empty), - Some("base64") => { - source - .get("media_type") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - || source - .get("data") - .and_then(Json::as_str) - .is_none_or(str::is_empty) - } - _ => true, + let response = response_fixture(protocol, true); + let decoded = + decode_response(&engine, protocol, &response).expect("response should decode"); + let encoded = + encode_response(&engine, protocol, &decoded).expect("response should encode"); + assert_eq!(encoded, response, "response mismatch for {protocol:?}"); + } + } + + #[test] + fn every_cross_protocol_pair_translates_common_text_response_data() { + let engine = translation_engine(); + for source in PROTOCOLS { + for target in PROTOCOLS { + if source == target { + continue; } - } else { - object.values().any(contains_invalid_anthropic_image_source) + + let request = request_fixture(source, false); + let decoded = decode_request( + &engine, + source, + &LlmRequest { + headers: Map::new(), + content: request, + }, + ) + .expect("request should decode"); + let encoded = encode_request(&engine, target, &decoded, Map::new()) + .expect("request should encode"); + assert!( + encoded.content.to_string().contains("hello"), + "translated request lost text for {source:?} -> {target:?}: {}", + encoded.content + ); + + let response = response_fixture(source, false); + let decoded = + decode_response(&engine, source, &response).expect("response should decode"); + let encoded = + encode_response(&engine, target, &decoded).expect("response should encode"); + let encoded_text = encoded.to_string(); + assert!( + encoded_text.contains("world"), + "translated response lost text for {source:?} -> {target:?}: {encoded}" + ); + assert!( + encoded_text.contains('7') && encoded_text.contains('3'), + "translated response lost usage for {source:?} -> {target:?}: {encoded}" + ); } } - Json::Array(items) => items.iter().any(contains_invalid_anthropic_image_source), - _ => false, } -} -fn contains_any_key_recursive(value: &Json, keys: &[&str]) -> bool { - match value { - Json::Object(object) => { - object.keys().any(|key| keys.contains(&key.as_str())) - || object - .values() - .any(|value| contains_any_key_recursive(value, keys)) + fn request_fixture(protocol: WireProtocol, extension: bool) -> Json { + let mut body = match protocol { + WireProtocol::OpenaiChat => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), + WireProtocol::OpenaiResponses => json!({ + "model": "caller/model", + "input": [{ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}] + }], + "max_output_tokens": 32 + }), + WireProtocol::AnthropicMessages => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), + }; + if extension { + body["provider_extension"] = json!({"exact": true}); } - Json::Array(items) => items - .iter() - .any(|value| contains_any_key_recursive(value, keys)), - _ => false, + body } -} -fn ensure_portable_response(protocol: WireProtocol, response: &Json) -> Result<()> { - let unsupported = - match protocol { - WireProtocol::OpenaiChat => { - response["choices"] - .as_array() - .into_iter() - .flatten() - .any(|choice| { - choice["message"].get("audio").is_some() - || choice["message"].get("reasoning_content").is_some() - }) - } - WireProtocol::OpenaiResponses => response["output"] - .as_array() - .into_iter() - .flatten() - .any(|item| { - matches!( - item.get("type").and_then(Json::as_str), - Some( - "reasoning" - | "computer_call" - | "computer_call_output" - | "web_search_call" - ) - ) - }), - WireProtocol::AnthropicMessages => response["content"] - .as_array() - .into_iter() - .flatten() - .any(|block| { - !matches!( - block.get("type").and_then(Json::as_str), - Some("text" | "tool_use") - ) - }), + fn response_fixture(protocol: WireProtocol, extension: bool) -> Json { + let mut body = match protocol { + WireProtocol::OpenaiChat => json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": "provider/model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "world"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 10} + }), + WireProtocol::OpenaiResponses => json!({ + "id": "resp-test", + "object": "response", + "model": "provider/model", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "world"}] + }], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10} + }), + WireProtocol::AnthropicMessages => json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": "provider/model", + "content": [{"type": "text", "text": "world"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 7, "output_tokens": 3} + }), }; - if unsupported { - Err(FlowError::InvalidArgument( - "provider-specific response extension cannot be translated safely".into(), - )) - } else { - Ok(()) + if extension { + body["provider_extension"] = json!({"exact": true}); + } + body } } diff --git a/crates/switchyard/tests/contract.rs b/crates/switchyard/tests/contract.rs deleted file mode 100644 index 7051f6a56..000000000 --- a/crates/switchyard/tests/contract.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for the public Switchyard Decision API contract. - -use nemo_relay_switchyard::contract::{DecisionAttempt, RoutingDecision}; -use serde_json::json; - -#[test] -fn current_switchyard_decision_contract_accepts_additive_fields() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"}, - "route": { - "tier": "capable", - "target_model": "model-a", - "backend_id": "backend-a", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "confidence": 0.8, - "future_field": {"safe": true} - })) - .unwrap(); - assert_eq!( - decision.extra.get("future_field"), - Some(&json!({"safe": true})) - ); - assert!(decision.baseline_route.is_none()); -} - -#[test] -fn current_switchyard_decision_contract_accepts_an_explicit_baseline() { - let decision: RoutingDecision = serde_json::from_value(json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage", "version": "1"}, - "route": { - "tier": "efficient", - "target_model": "model-small", - "backend_id": "backend-small", - "target_protocol_profile": "openai_chat", - "target_endpoint": "/v1/chat/completions" - }, - "baseline_route": { - "tier": "capable", - "target_model": "model-large", - "backend_id": "backend-large", - "target_protocol_profile": "anthropic_messages", - "target_endpoint": "/v1/messages" - } - })) - .unwrap(); - let baseline = decision.baseline_route.unwrap(); - assert_eq!(baseline.backend_id, "backend-large"); - assert_eq!(baseline.target_protocol_profile, "anthropic_messages"); -} - -#[test] -fn malformed_decisions_are_rejected_by_serde() { - let missing_route = json!({ - "schema_version": "switchyard.routing_decision.v1", - "decision_id": "decision-1", - "router": {"name": "stage_router", "version": "1"} - }); - assert!(serde_json::from_value::(missing_route).is_err()); -} - -#[test] -fn additive_retry_metadata_stays_optional_on_the_wire() { - let attempt: DecisionAttempt = serde_json::from_value(json!({ - "routing_attempt": 1, - "max_routing_attempts": 4 - })) - .unwrap(); - assert!(attempt.previous_route.is_none()); - assert!(attempt.retry_reason.is_none()); -} diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs index 670d9159b..c89321aa4 100644 --- a/crates/switchyard/tests/unit/component_tests.rs +++ b/crates/switchyard/tests/unit/component_tests.rs @@ -1,1827 +1,1138 @@ // SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Unit tests for the Switchyard Relay plugin component. - -use std::future::Future; -use std::pin::Pin; -use std::sync::Mutex; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::task::{Context, Poll}; - -use axum::{ - Json as AxumJson, Router, - extract::State, - http::StatusCode, - routing::{get, post}, -}; -use futures_util::{Stream, stream as futures_stream}; -use nemo_relay::api::event::{Event, ScopeCategory}; -use nemo_relay::api::llm::{ - LlmCallExecuteParams, LlmStreamCallExecuteParams, llm_call_execute, llm_stream_call_execute, -}; -use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmStreamExecutionNextFn, LlmStreamInner}; -use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; -use nemo_relay::codec::optimization::LlmOptimizationSummaryStatus; -use nemo_relay::error::{UpstreamFailure, UpstreamFailureClass}; -use nemo_relay::plugin::rollback_registrations; - -use super::*; - -struct CloseTrackingStream { - close_calls: Arc, - closed: bool, -} - -impl Stream for CloseTrackingStream { - type Item = FlowResult; +use std::sync::{Arc, Mutex}; - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(None) - } -} - -impl LlmStreamInner for CloseTrackingStream { - fn close( - mut self: Pin<&mut Self>, - ) -> Pin> + Send + '_>> { - self.closed = true; - self.close_calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok(()) }) - } -} +use futures_util::{StreamExt, stream}; +use nemo_relay::api::llm::LlmRequest; +use nemo_relay::api::runtime::{LlmExecutionNextFn, LlmJsonStream, LlmStreamExecutionNextFn}; +use nemo_relay::error::{FlowError, UpstreamFailure, UpstreamFailureClass}; +use serde_json::{Map, Value as Json, json}; +use switchyard_libsy::{LlmResponse, Response as LibsyResponse, Step}; +use switchyard_protocol::text_response; -fn close_tracking_stream(close_calls: Arc) -> LlmJsonStream { - LlmJsonStream::from_closeable(CloseTrackingStream { - close_calls, - closed: false, - }) -} - -fn prefixed_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - LlmJsonStream::from_closeable(PrefixedStream { - first: Some(Ok(json!({"first": true}))), - upstream, - }) -} +use super::*; -fn terminal_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - mark_terminal_stream(upstream, "test", "enforce", json!({"route": "test"})) -} +const CLASSIFIER_VERDICT: &str = r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.95,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#; -fn translated_adapter(upstream: LlmJsonStream) -> LlmJsonStream { - translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - upstream, - ) -} - -fn binding(protocol: WireProtocol, model: &str) -> TargetBinding { +fn binding(protocol: WireProtocol, model: &str, weight: f64) -> TargetBinding { TargetBinding { model: model.into(), protocol, endpoint: protocol.endpoint().into(), - base_url: "http://127.0.0.1:9999".into(), + base_url: "https://provider.example.com/v1".into(), + weight, headers: BTreeMap::new(), header_env: BTreeMap::new(), } } -fn config(decision_api_url: String) -> SwitchyardConfig { +fn chat_config() -> SwitchyardConfig { + SwitchyardConfig { + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([( + "chat".into(), + binding(WireProtocol::OpenaiChat, "provider/chat", 1.0), + )]), + default_targets: ProtocolDefaults { + openai_chat: "chat".into(), + ..ProtocolDefaults::default() + }, + enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiChat]), + ..SwitchyardConfig::default() + } +} + +fn classifier_config(session_affinity: bool) -> SwitchyardConfig { SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, + algorithm: AlgorithmConfig::LlmClassifier { + classifier_target: "classifier".into(), + weak_target: "weak".into(), + strong_target: "strong".into(), + base_threshold: 0.5, + min_confidence: 0.5, + capability_elevated_floor: Some(0.8), + session_affinity, + message_hash_fallback: false, + }, targets: BTreeMap::from([ ( - "selected-chat".into(), - binding(WireProtocol::OpenaiChat, "selected"), - ), - ( - "baseline-chat".into(), - binding(WireProtocol::OpenaiChat, "baseline"), - ), - ( - "fallback-chat".into(), - binding(WireProtocol::OpenaiChat, "fallback"), + "classifier".into(), + binding(WireProtocol::OpenaiChat, "provider/classifier", 1.0), ), ( - "fallback-responses".into(), - binding(WireProtocol::OpenaiResponses, "fallback"), + "weak".into(), + binding(WireProtocol::OpenaiChat, "provider/weak", 1.0), ), ( - "fallback-anthropic".into(), - binding(WireProtocol::AnthropicMessages, "fallback"), + "strong".into(), + binding(WireProtocol::OpenaiChat, "provider/strong", 1.0), ), ]), default_targets: ProtocolDefaults { - openai_chat: "fallback-chat".into(), - openai_responses: "fallback-responses".into(), - anthropic_messages: "fallback-anthropic".into(), + openai_chat: "strong".into(), + ..ProtocolDefaults::default() }, + enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiChat]), ..SwitchyardConfig::default() } } -fn decision() -> RoutingDecision { - RoutingDecision { - schema_version: ROUTING_DECISION_SCHEMA_VERSION.into(), - decision_id: "decision-1".into(), - router: crate::contract::DecisionProvider { - name: "stage_router".into(), - version: "1".into(), - }, - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "selected".into(), - backend_id: "selected-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }, - baseline_route: Some(crate::contract::RoutingTarget { - tier: "capable".into(), - target_model: "baseline".into(), - backend_id: "baseline-chat".into(), - target_protocol_profile: "openai_chat".into(), - target_endpoint: "/v1/chat/completions".into(), - }), - confidence: Some(0.9), - reason_code: Some("test".into()), - reason_summary: None, - metadata: BTreeMap::from([ - ("feature_state".into(), json!("fresh")), - ("snapshot_age_millis".into(), json!(37)), - ("snapshot_max_age_millis".into(), json!(300_000)), - ]), - extra: BTreeMap::new(), - } +async fn run_buffered_classifier(classifier_protocol: WireProtocol) -> (Json, Vec) { + let mut config = classifier_config(false); + config.targets.insert( + "classifier".into(), + binding( + classifier_protocol, + match classifier_protocol { + WireProtocol::OpenaiChat => "provider/classifier", + WireProtocol::OpenaiResponses => "provider/classifier-responses", + WireProtocol::AnthropicMessages => unreachable!(), + }, + 1.0, + ), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request.clone()); + if model.starts_with("provider/classifier") { + return Ok(match classifier_protocol { + WireProtocol::OpenaiChat => { + assert!(request.content["response_format"].is_object()); + chat_response(&model, CLASSIFIER_VERDICT) + } + WireProtocol::OpenaiResponses => { + assert!(request.content["text"]["format"].is_object()); + responses_response(&model, CLASSIFIER_VERDICT) + } + WireProtocol::AnthropicMessages => unreachable!(), + }); + } + Ok(chat_response(&model, "efficient answer")) + }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + let requests = requests.lock().unwrap().clone(); + (response, requests) } fn chat_request() -> LlmRequest { LlmRequest { - headers: Map::new(), + headers: Map::from_iter([ + ( + "x-nemo-relay-session-id".into(), + Json::String("session-1".into()), + ), + ( + "authorization".into(), + Json::String("Bearer caller-secret".into()), + ), + ]), content: json!({ - "model": "inbound", - "messages": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "answer"}, - {"role": "user", "content": "latest"} - ] + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"preserve": true} }), } } -fn request(protocol: WireProtocol) -> LlmRequest { - let content = match protocol { - WireProtocol::OpenaiChat => chat_request().content, - WireProtocol::OpenaiResponses => { - json!({"model": "inbound", "instructions": "system", "input": "latest"}) - } - WireProtocol::AnthropicMessages => { - json!({"model": "inbound", "system": "system", "max_tokens": 32, "messages": [{"role": "user", "content": "latest"}]}) +fn chat_response(model: &str, text: &str) -> Json { + json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": model, + "system_fingerprint": "fp_exact", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 } - }; - LlmRequest { - headers: Map::new(), - content, - } + }) } -fn chat_response() -> Json { +fn chat_chunk(model: &str, text: &str) -> Json { json!({ - "id": "chat-1", "object": "chat.completion", "model": "selected", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "model": model, + "system_fingerprint": "fp_stream_exact", + "choices": [{ + "index": 0, + "delta": {"content": text}, + "finish_reason": null + }] }) } -fn chat_chunk(text: &str, finish_reason: Json) -> Json { +fn responses_response(model: &str, text: &str) -> Json { json!({ - "id": "chat-1", "object": "chat.completion.chunk", "model": "selected", - "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": finish_reason}] + "id": "resp-test", + "object": "response", + "status": "completed", + "model": model, + "output": [{ + "type": "message", + "id": "msg-test", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} }) } -#[test] -fn wire_protocol_and_plugin_lifecycle_contracts_are_stable() { - for (protocol, label, endpoint) in [ - ( - WireProtocol::OpenaiChat, - "openai_chat", - "/v1/chat/completions", - ), - ( - WireProtocol::OpenaiResponses, - "openai_responses", - "/v1/responses", - ), - ( - WireProtocol::AnthropicMessages, - "anthropic_messages", - "/v1/messages", - ), - ] { - assert_eq!(protocol.label(), label); - assert_eq!(protocol.endpoint(), endpoint); - assert_eq!( - WireProtocol::from_call(label, &request(protocol)), - Some(protocol) - ); - assert_eq!( - WireProtocol::from_call("unknown", &request(protocol)), - Some(protocol) - ); - } - assert_eq!( - WireProtocol::from_call( - "unknown", - &LlmRequest { - headers: Map::new(), - content: json!({}) - } - ), - None - ); - assert_eq!(RoutingMode::Enforce.label(), "enforce"); - assert_eq!(RoutingMode::ObserveOnly.label(), "observe_only"); - - let valid = config("http://127.0.0.1:1/v1/routing/decision".into()); - let component: PluginComponentSpec = valid.clone().into(); - assert_eq!(component.kind, SWITCHYARD_PLUGIN_KIND); - assert!(component.enabled); - assert_eq!(component.config["decision_profile_id"], "stage_router"); - - let plugin = SwitchyardPlugin; - assert_eq!(plugin.plugin_kind(), SWITCHYARD_PLUGIN_KIND); - assert!(!plugin.allows_multiple_components()); - assert!(plugin.validate(&component.config).is_empty()); - let diagnostics = plugin.validate(json!({"version": "invalid"}).as_object().unwrap()); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].level, DiagnosticLevel::Error); - assert_eq!(diagnostics[0].code, "switchyard.invalid_config"); -} - -#[tokio::test] -async fn plugin_registration_installs_and_rolls_back_both_execution_intercepts() { - let plugin = SwitchyardPlugin; - let decision_api_url = health_server(StatusCode::OK, r#"{"status":"ok"}"#).await; - for (mode, profile_id) in [ - (RoutingMode::Enforce, "random"), - (RoutingMode::Enforce, "llm"), - (RoutingMode::Enforce, "stage_router"), - (RoutingMode::ObserveOnly, "random"), - (RoutingMode::ObserveOnly, "llm"), - (RoutingMode::ObserveOnly, "stage_router"), - ] { - let mut candidate = config(decision_api_url.clone()); - candidate.mode = mode; - candidate.decision_profile_id = profile_id.into(); - let component: PluginComponentSpec = candidate.into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_retries_transient_sidecar_health_failures() { - let (decision_api_url, attempts) = recovering_health_server(2).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - plugin - .register(&component.config, &mut context) - .await - .unwrap(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - let mut registrations = context.into_registrations(); - assert_eq!(registrations.len(), 2); - rollback_registrations(&mut registrations); - assert!(registrations.is_empty()); -} - -#[tokio::test] -async fn plugin_registration_returns_the_final_health_failure_after_retry_exhaustion() { - let (decision_api_url, attempts) = recovering_health_server(usize::MAX).await; - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(decision_api_url).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert!( - error - .to_string() - .contains("returned HTTP 503 Service Unavailable") - ); - assert!(context.into_registrations().is_empty()); -} - -#[tokio::test] -async fn plugin_registration_rejects_unhealthy_or_invalid_sidecars() { - for (status, body, expected) in [ - ( - StatusCode::SERVICE_UNAVAILABLE, - r#"{"status":"starting"}"#, - "returned HTTP 503 Service Unavailable", - ), - (StatusCode::OK, "not-json", "returned invalid JSON"), - ( - StatusCode::OK, - r#"{"status":"starting"}"#, - "did not report status=ok", - ), - ] { - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = config(health_server(status, body).await).into(); - let mut context = PluginRegistrationContext::with_namespace(format!( - "switchyard-test-{}-", - Uuid::now_v7() - )); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!( - error.to_string().contains(expected), - "{error:?} did not contain {expected:?}" - ); - assert!(context.into_registrations().is_empty()); - } -} - -#[tokio::test] -async fn plugin_registration_rejects_an_unreachable_sidecar() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - - let plugin = SwitchyardPlugin; - let component: PluginComponentSpec = - config(format!("http://{address}/v1/routing/decision")).into(); - let mut context = - PluginRegistrationContext::with_namespace(format!("switchyard-test-{}-", Uuid::now_v7())); - let error = plugin - .register(&component.config, &mut context) - .await - .unwrap_err(); - assert!(error.to_string().contains("Switchyard service is required")); - assert!(error.to_string().contains("/health")); - assert!(context.into_registrations().is_empty()); +fn anthropic_response(model: &str, text: &str) -> Json { + json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }) } -#[test] -fn configuration_validation_rejects_unsafe_or_ambiguous_bindings() { - let url = "http://127.0.0.1:1/v1/routing/decision".to_string(); - let assert_invalid = |config: SwitchyardConfig, expected: &str| { - let error = SwitchyardRuntime::new(config) - .err() - .expect("config must fail"); - assert!( - error.contains(expected), - "{error:?} did not contain {expected:?}" - ); +fn request_for(protocol: WireProtocol) -> LlmRequest { + let mut request = chat_request(); + request.content = match protocol { + WireProtocol::OpenaiChat => request.content, + WireProtocol::OpenaiResponses => json!({ + "model": "caller/model", + "input": [{ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}] + }] + }), + WireProtocol::AnthropicMessages => json!({ + "model": "caller/model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 32 + }), }; + request +} - let mut candidate = config(url.clone()); - candidate.version = 2; - assert_invalid(candidate, "unsupported Switchyard config version"); - let mut candidate = config(url.clone()); - candidate.decision_profile_id.clear(); - assert_invalid(candidate, "decision_profile_id"); - let mut candidate = config(url.clone()); - candidate.decision_timeout_millis = 0; - assert_invalid(candidate, "decision_timeout_millis"); - let mut candidate = config(url.clone()); - candidate.max_retries = 11; - assert_invalid(candidate, "max_retries"); - let mut candidate = config(url.clone()); - candidate.recent_message_count = 0; - assert_invalid(candidate, "recent_message_count"); - let mut candidate = config(url.clone()); - candidate.context_mode = ContextMode::AtofRequired; - assert_invalid(candidate, "atof_endpoint_name"); - let mut candidate = config(url.clone()); - candidate.atof_endpoint_name = Some(" switchyard ".into()); - assert_invalid(candidate, "leading or trailing whitespace"); - let candidate = config("file:///tmp/decision".into()); - assert_invalid(candidate, "must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.clear(); - assert_invalid(candidate, "targets must not be empty"); - let mut candidate = config(url.clone()); - candidate.enabled_inbound_profiles.clear(); - assert_invalid(candidate, "enabled_inbound_profiles"); - - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().endpoint = "/wrong".into(); - assert_invalid(candidate, "endpoint must be"); - let mut candidate = config(url.clone()); - candidate.targets.get_mut("selected-chat").unwrap().base_url = "file:///tmp/provider".into(); - assert_invalid(candidate, "base_url must use http or https"); - let mut candidate = config(url.clone()); - candidate.targets.insert( - "duplicate-chat".into(), - candidate.targets["selected-chat"].clone(), - ); - assert_invalid(candidate, "conflicts with another exact backend binding"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "missing".into(); - assert_invalid(candidate, "default target \"missing\" is not configured"); - let mut candidate = config(url.clone()); - candidate.default_targets.openai_chat = "fallback-responses".into(); - assert_invalid(candidate, "must use protocol openai_chat"); - - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("bad header".into(), "value".into()); - assert_invalid(candidate, "invalid header name"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("x-test".into(), "bad\nvalue".into()); - assert_invalid(candidate, "invalid header value"); - let mut candidate = config(url.clone()); - candidate - .decision_headers - .insert("Authorization".into(), "Bearer static".into()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url.clone()); - candidate.decision_header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_DECISION_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); - let mut candidate = config(url.clone()); - let target = candidate.targets.get_mut("selected-chat").unwrap(); - target - .headers - .insert("Authorization".into(), "Bearer static".into()); - target.header_env.insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "cannot appear in both headers and header_env"); - let mut candidate = config(url); - candidate - .targets - .get_mut("selected-chat") - .unwrap() - .header_env - .insert( - "authorization".into(), - "SWITCHYARD_TEST_MISSING_TARGET_SECRET".into(), - ); - assert_invalid(candidate, "is not set"); +fn provider_error(class: UpstreamFailureClass, status: Option) -> FlowError { + FlowError::Upstream(UpstreamFailure { + status, + body: "provider failure".into(), + headers: BTreeMap::new(), + class, + }) } #[test] -fn disabled_protocol_defaults_are_optional() { - let defaults: ProtocolDefaults = serde_json::from_value(json!({ - "anthropic_messages": "fallback-anthropic" - })) - .expect("omitted disabled protocol defaults should deserialize"); - assert!(defaults.openai_chat.is_empty()); - assert!(defaults.openai_responses.is_empty()); - - let mut candidate = config("http://127.0.0.1:1/v1/routing/decision".into()); - candidate.enabled_inbound_profiles = BTreeSet::from([WireProtocol::AnthropicMessages]); - candidate.default_targets.openai_chat.clear(); - candidate.default_targets.openai_responses.clear(); - - SwitchyardRuntime::new(candidate).expect("Anthropic-only configuration should validate"); +fn version_one_configuration_has_a_migration_error() { + let error = parse_config(&Map::from_iter([("version".into(), json!(1))])).unwrap_err(); + assert!(error.contains("removed switchyard-server Decision API")); + assert!(error.contains("version = 2")); } #[test] -fn atof_cross_component_validation_rejects_invalid_endpoint_names_before_lookup() { - for (name, expected) in [ - (" ", "atof_endpoint_name must be non-empty when configured"), - ( - " switchyard ", - "atof_endpoint_name must not have leading or trailing whitespace", - ), - ] { - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some(name.into()); - let plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - - assert_eq!( - validate_switchyard_atof_configuration(&plugin_config).unwrap_err(), - expected - ); - } +fn random_configuration_rejects_invalid_targets_and_weights() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().endpoint = "/wrong".into(); + assert!(validate_config(&config).is_err()); + + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = f64::NAN; + assert!(validate_config(&config).is_err()); + + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + assert!(validate_config(&config).is_err()); } #[test] -fn atof_cross_component_validation_reports_each_activation_mismatch() { - assert!(validate_switchyard_atof_configuration(&PluginConfig::default()).is_ok()); +fn classifier_configuration_validates_targets_thresholds_and_translation_support() { + assert!(validate_config(&classifier_config(false)).is_ok()); - let payload = PluginConfig { - components: vec![config("http://switchyard.test/v1/routing/decision".into()).into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&payload).is_ok()); - - let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": {"enabled": true, "sinks": [{ - "type": "stream", - "name": "other", - "url": "http://events.test/v1/atof/events", - "header_env": {"authorization": "TOKEN"} - }]}}) - .as_object() - .unwrap() - .clone(), - }); + let mut config = classifier_config(false); + config.targets.remove("classifier"); assert!( - validate_switchyard_atof_configuration(&plugin_config) + validate_config(&config) .unwrap_err() - .contains("requires named ATOF endpoint") + .contains("algorithm target \"classifier\" is not configured") ); - plugin_config.components[1].config["atof"]["sinks"][0]["name"] = json!("switchyard"); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("websocket"); - assert!( - validate_switchyard_atof_configuration(&plugin_config) - .unwrap_err() - .contains("transport = http_post") - ); - plugin_config.components[1].config["atof"]["sinks"][0]["transport"] = json!("http_post"); - let duplicate = plugin_config.components[1].config["atof"]["sinks"][0].clone(); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .push(duplicate); + let mut config = classifier_config(false); + if let AlgorithmConfig::LlmClassifier { base_threshold, .. } = &mut config.algorithm { + *base_threshold = 1.5; + } assert!( - validate_switchyard_atof_configuration(&plugin_config) + validate_config(&config) .unwrap_err() - .contains("exactly one endpoint") + .contains("base_threshold must be between 0 and 1") ); - plugin_config.components[1].config["atof"]["sinks"] - .as_array_mut() - .unwrap() - .pop(); - plugin_config.components[1].config["atof"]["sinks"][0]["field_name_policy"] = - json!("snake_case"); + + let mut config = classifier_config(false); + let classifier = config.targets.get_mut("classifier").unwrap(); + classifier.protocol = WireProtocol::AnthropicMessages; + classifier.endpoint = WireProtocol::AnthropicMessages.endpoint().into(); assert!( - validate_switchyard_atof_configuration(&plugin_config) + validate_config(&config) .unwrap_err() - .contains("field_name_policy = preserve") + .contains("structured response format") ); - let endpoint = &mut plugin_config.components[1].config["atof"]["sinks"][0]; - endpoint["field_name_policy"] = json!("preserve"); - endpoint.as_object_mut().unwrap().remove("header_env"); + + let mut config = classifier_config(false); + if let AlgorithmConfig::LlmClassifier { + message_hash_fallback, + .. + } = &mut config.algorithm + { + *message_hash_fallback = true; + } assert!( - validate_switchyard_atof_configuration(&plugin_config) + validate_config(&config) .unwrap_err() - .contains("environment-referenced header") + .contains("message_hash_fallback requires session_affinity") ); } #[tokio::test] -async fn execution_bypasses_inapplicable_calls_and_fails_open_on_extensions() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let passthrough: LlmExecutionNextFn = - Arc::new(|request| Box::pin(async move { Ok(request.content) })); - let unknown = LlmRequest { - headers: Map::new(), - content: json!({"opaque": true}), - }; - assert_eq!( - runtime - .execute_buffered("custom.provider", unknown, Arc::clone(&passthrough)) - .await - .unwrap()["opaque"], - true - ); - - let mut disabled_config = config("http://127.0.0.1:1/v1/routing/decision".into()); - disabled_config - .enabled_inbound_profiles - .remove(&WireProtocol::OpenaiChat); - let disabled = SwitchyardRuntime::new(disabled_config).unwrap(); - assert_eq!( - disabled - .execute_buffered( - "openai.chat_completions", - chat_request(), - Arc::clone(&passthrough), - ) - .await - .unwrap()["model"], - "inbound" - ); - - let mut unsupported = chat_request(); - unsupported.content["thinking"] = json!({"type": "enabled"}); - let response = runtime - .execute_buffered( - "openai.chat_completions", - unsupported, - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await +async fn run_stream_requires_relay_to_fulfill_the_provider_call() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let request = runtime + .libsy_request(WireProtocol::OpenaiChat, &chat_request(), false) .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); -} + let mut steps = runtime + .algorithm + .clone() + .run_stream(LibsyContext::default(), request); -#[tokio::test] -async fn stream_setup_retries_and_empty_streams_have_one_bounded_fallback() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connect".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })); - } - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); - - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmStreamExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt < 4 { - Vec::new() - } else { - assert_eq!(request.content["model"], "fallback"); - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }); - let output = runtime - .execute_stream("openai.chat_completions", chat_request(), next) - .await - .unwrap() - .collect::>() - .await; - assert_eq!(output.len(), 1); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - assert_eq!(decisions.lock().unwrap().len(), 4); -} + let decision = steps.next().await.unwrap().unwrap(); + assert!(matches!(decision, Step::Decision(_))); -#[tokio::test] -async fn fallback_setup_failures_preserve_the_provider_error() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let buffered: LlmExecutionNextFn = Arc::new(|_| { - Box::pin(async { Err(FlowError::Internal("buffered fallback failed".into())) }) - }); - let error = runtime - .dispatch_fallback_buffered(WireProtocol::OpenaiChat, chat_request(), buffered, "test") - .await - .unwrap_err(); - assert!(error.to_string().contains("buffered fallback failed")); - - let streaming: LlmStreamExecutionNextFn = - Arc::new(|_| Box::pin(async { Err(FlowError::Internal("stream fallback failed".into())) })); - let error = runtime - .dispatch_fallback_stream(WireProtocol::OpenaiChat, chat_request(), streaming, "test") - .await - .err() - .expect("stream fallback setup must fail"); - assert!(error.to_string().contains("stream fallback failed")); -} + let call = steps.next().await.unwrap().unwrap(); + let Step::CallLlm(call) = call else { + panic!("expected Relay-hosted CallLlm step"); + }; + assert_eq!(call.get_decision().selected_model(), "chat"); + call.respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response(Some("provider/chat".into()), "hello")), + metadata: None, + })) + .unwrap(); -#[tokio::test] -async fn translated_stream_preserves_success_and_propagates_both_error_sources() { - let source = LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("hello", Json::Null)), - Ok(chat_chunk("", json!("stop"))), - ])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output.iter().all(Result::is_ok)); - assert!(output.iter().any(|item| { - item.as_ref() - .is_ok_and(|chunk| chunk.to_string().contains("hello")) - })); - - let upstream_error = FlowError::Internal("upstream stream failed".into()); - let source = LlmJsonStream::new(futures_stream::iter(vec![Err(upstream_error)])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - source, - ) - .collect::>() - .await; - assert!(output[0].is_err()); - - let malformed = LlmJsonStream::new(futures_stream::iter(vec![Ok(json!({ - "choices": [{"delta": {"reasoning_content": "private"}}] - }))])); - let output = translated_stream( - WireProtocol::OpenaiChat, - WireProtocol::AnthropicMessages, - "selected".into(), - malformed, - ) - .collect::>() - .await; - assert!(output[0].is_err()); + let returned = steps.next().await.unwrap().unwrap(); + assert!(matches!(returned, Step::ReturnToAgent(_))); } #[tokio::test] -async fn stream_adapters_forward_explicit_close_to_the_upstream_stream() { - for make_adapter in [ - prefixed_adapter as fn(LlmJsonStream) -> LlmJsonStream, - terminal_adapter, - translated_adapter, - ] { - let close_calls = Arc::new(AtomicUsize::new(0)); - let mut stream = make_adapter(close_tracking_stream(Arc::clone(&close_calls))); - - stream.close().await.unwrap(); - - assert_eq!(close_calls.load(Ordering::SeqCst), 1); - assert!(stream.next().await.is_none()); - } -} - -#[test] -fn provider_failure_reporting_covers_every_retry_class() { - for (class, label, retryable) in [ - (UpstreamFailureClass::Connection, "connection", true), - (UpstreamFailureClass::Timeout, "timeout", true), - ( - UpstreamFailureClass::RetryableStatus, - "retryable_status", - true, - ), - (UpstreamFailureClass::ContextWindow, "context_window", true), - ( - UpstreamFailureClass::ModelUnavailable, - "model_unavailable", - true, - ), - ( - UpstreamFailureClass::Authentication, - "authentication", - false, - ), - ( - UpstreamFailureClass::InvalidRequest, - "invalid_request", - false, - ), - (UpstreamFailureClass::Other, "other", false), - ] { - let error = FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "failure".into(), - headers: BTreeMap::new(), - class, - }); - assert_eq!(provider_error_class(&error), label); - assert_eq!(error_is_retryable(&error), retryable); - assert_eq!(provider_error_summary(&error), format!("{label}:http_503")); - } - let relay = FlowError::Internal("failure".into()); - assert_eq!(provider_error_class(&relay), "relay"); - assert_eq!(provider_error_summary(&relay), relay.to_string()); -} - -#[test] -fn all_materialization_modes_are_bounded_and_provider_valid() { - for protocol in [ - WireProtocol::OpenaiChat, - WireProtocol::OpenaiResponses, - WireProtocol::AnthropicMessages, - ] { - for mode in [ - RequestMaterialization::None, - RequestMaterialization::SummaryOnly, - RequestMaterialization::LatestUserPrompt, - RequestMaterialization::RecentMessageWindow, - RequestMaterialization::AnnotatedRequest, - RequestMaterialization::FullBody, - ] { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - config.request_materialization = mode; - config.recent_message_count = 2; - let runtime = SwitchyardRuntime::new(config).unwrap(); - let routing = runtime - .routing_request(protocol, &request(protocol), 1, None) - .unwrap(); - match mode { - RequestMaterialization::None | RequestMaterialization::SummaryOnly => { - assert!(routing.current_request.is_none()) - } - RequestMaterialization::LatestUserPrompt => { - let current = routing.current_request.unwrap(); - assert_eq!(current["latest_user_prompt"], "latest"); - let body = current["body"].clone(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: body, - }, - ) - .unwrap(); - } - RequestMaterialization::RecentMessageWindow => { - let current = routing.current_request.unwrap(); - decode_request( - &runtime.translation, - protocol, - &LlmRequest { - headers: Map::new(), - content: current["body"].clone(), - }, - ) - .unwrap(); - } - RequestMaterialization::AnnotatedRequest | RequestMaterialization::FullBody => { - assert!(routing.current_request.is_some()) - } - } - } - } -} - -#[test] -fn identity_policy_requires_stable_request_scope_only_for_atof_profiles() { - let mut config = config("http://127.0.0.1:1/v1/routing/decision".into()); - let payload_runtime = SwitchyardRuntime::new(config.clone()).unwrap(); - let synthetic = payload_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) +async fn classifier_run_stream_offloads_the_judge_and_routed_calls_to_relay() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let request = runtime + .libsy_request(WireProtocol::OpenaiChat, &chat_request(), false) .unwrap(); - assert_eq!(synthetic.identity.quality, "synthetic"); + let mut steps = runtime + .algorithm + .clone() + .run_stream(LibsyContext::default(), request); - config.context_mode = ContextMode::AtofRequired; - config.atof_endpoint_name = Some("switchyard".into()); - let atof_runtime = SwitchyardRuntime::new(config).unwrap(); + let Step::CallLlm(classifier) = steps.next().await.unwrap().unwrap() else { + panic!("expected classifier CallLlm before the routing decision"); + }; + assert_eq!(classifier.get_decision().selected_model(), "classifier"); + assert!(!classifier.get_decision().is_routed_call()); assert!( - atof_runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .is_err() + classifier + .get_request() + .llm_request + .output + .response_format + .is_some() ); - let mut stable = chat_request(); - stable - .headers - .insert("x-nemo-relay-session-id".into(), json!("session-1")); - stable - .headers - .insert("x-nemo-relay-request-id".into(), json!("request-1")); - let routed = atof_runtime - .routing_request(WireProtocol::OpenaiChat, &stable, 1, None) + classifier + .respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response( + Some("provider/classifier".into()), + CLASSIFIER_VERDICT, + )), + metadata: None, + })) .unwrap(); - assert_eq!(routed.identity.quality, "explicit"); -} - -#[test] -fn exact_target_validation_rejects_any_switchyard_drift() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - assert!(runtime.validate_decision(&decision()).is_ok()); - let mut drifted = decision(); - drifted.route.target_model = "unbound-model".into(); - assert!(runtime.validate_decision(&drifted).is_err()); - drifted = decision(); - drifted.route.backend_id = "unknown".into(); - assert!(runtime.validate_decision(&drifted).is_err()); -} - -#[test] -fn routing_contribution_requires_an_exact_independent_baseline_binding() { - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - - let contribution = runtime.routing_contribution(&decision(), 2, true).unwrap(); - assert!(contribution.applied); - assert_eq!( - contribution.kind.as_str(), - LlmOptimizationKind::MODEL_ROUTING - ); - let transition = contribution.model_transition.unwrap(); - assert_eq!(transition.baseline.unwrap().model, "baseline"); - assert_eq!(transition.effective.unwrap().model, "selected"); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["feature_state"], - "fresh" - ); - assert_eq!( - contribution.payload.as_ref().unwrap()["router_metadata"]["snapshot_age_millis"], - 37 - ); - assert_eq!( - contribution.payload_schema.as_ref().unwrap().name, - ROUTING_CONTRIBUTION_SCHEMA - ); - let observed = runtime.routing_contribution(&decision(), 1, false).unwrap(); - assert!(!observed.applied); + let Step::Decision(decision) = steps.next().await.unwrap().unwrap() else { + panic!("expected final routing decision after the classifier response"); + }; + assert_eq!(decision.selected_model(), "weak"); + assert_eq!(decision.routing_tier(), Some("weak")); + assert!(decision.is_routed_call()); - let mut missing = decision(); - missing.baseline_route = None; - assert!(runtime.routing_contribution(&missing, 1, true).is_none()); + let Step::CallLlm(routed) = steps.next().await.unwrap().unwrap() else { + panic!("expected routed CallLlm"); + }; + assert_eq!(routed.get_decision().selected_model(), "weak"); + routed + .respond(Ok(LibsyResponse { + llm_response: LlmResponse::Agg(text_response(Some("provider/weak".into()), "answer")), + metadata: None, + })) + .unwrap(); - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - assert!(runtime.routing_contribution(&drifted, 1, true).is_none()); - assert!(runtime.validate_decision(&drifted).is_ok()); + assert!(matches!( + steps.next().await.unwrap().unwrap(), + Step::ReturnToAgent(_) + )); } -#[test] -fn routing_decision_mark_has_canonical_shape_and_mirrored_identity() { - let subscriber_name = format!("switchyard-mark-shape-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - - let runtime = - SwitchyardRuntime::new(config("http://127.0.0.1:1/v1/routing/decision".into())).unwrap(); - let routing_request = runtime - .routing_request(WireProtocol::OpenaiChat, &chat_request(), 1, None) - .unwrap(); - runtime.emit_decision(&routing_request, &decision(), 1, false, 17); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); +#[tokio::test] +async fn buffered_classifier_dispatches_the_judge_then_the_selected_target() { + let (response, requests) = run_buffered_classifier(WireProtocol::OpenaiChat).await; + assert_eq!( + response["choices"][0]["message"]["content"], + "efficient answer" + ); - let event = events - .lock() - .unwrap() + let models = requests .iter() - .map(Event::to_json_value) - .find(|event| { - event["name"] == "switchyard.routing.decision" - && event["metadata"]["session_id"] == routing_request.identity.session_id - && event["metadata"]["request_id"] == routing_request.identity.request_id - }) - .expect("decision mark should be captured"); - assert_eq!(event["kind"], "mark"); - assert_eq!(event["category"], "custom"); + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!(models, vec!["provider/classifier", "provider/weak"]); assert_eq!( - event["category_profile"]["subtype"], - "switchyard.routing.decision" + requests[1].content["provider_extension"], + json!({"preserve": true}) ); - assert_eq!(event["data_schema"]["name"], ROUTING_MARK_SCHEMA); - assert_eq!(event["data_schema"]["version"], "1"); - assert_eq!(event["data"]["profile_id"], "stage_router"); - assert_eq!(event["data"]["selected_model"], "selected"); - assert_eq!(event["data"]["latency_ms"], 17); - assert_eq!(event["data"]["router_metadata"]["feature_state"], "fresh"); - assert_eq!(event["data"]["router_metadata"]["snapshot_age_millis"], 37); +} + +#[tokio::test] +async fn buffered_classifier_supports_an_openai_responses_judge_target() { + let (response, requests) = run_buffered_classifier(WireProtocol::OpenaiResponses).await; assert_eq!( - event["metadata"]["session_id"], - routing_request.identity.session_id + response["choices"][0]["message"]["content"], + "efficient answer" ); + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); assert_eq!( - event["metadata"]["request_id"], - routing_request.identity.request_id + models, + vec!["provider/classifier-responses", "provider/weak"] ); } -#[test] -fn atof_required_cross_component_validation_is_context_sensitive() { - let mut switchyard = config("http://switchyard.test:8080/v1/routing/decision".into()); - switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_name = Some("switchyard".into()); - let mut plugin_config = PluginConfig { - components: vec![switchyard.into()], - ..PluginConfig::default() - }; - assert!(validate_switchyard_atof_configuration(&plugin_config).is_err()); - plugin_config.components.push(PluginComponentSpec { - kind: "observability".into(), - enabled: true, - config: json!({"atof": { - "enabled": true, - "sinks": [{ - "type": "stream", - "name": "switchyard", - "url": "http://switchyard.test:8080/v1/atof/events", - "transport": "http_post", - "field_name_policy": "preserve", - "header_env": {"authorization": "SWITCHYARD_TOKEN"} - }] - }}) - .as_object() - .unwrap() - .clone(), - }); - assert!(validate_switchyard_atof_configuration(&plugin_config).is_ok()); -} - -#[derive(Clone)] -struct DecisionState { - requests: Arc>>, - decision: RoutingDecision, -} +#[tokio::test] +async fn classifier_prompt_is_encoded_as_a_system_instruction() { + let (_, chat_requests) = run_buffered_classifier(WireProtocol::OpenaiChat).await; + assert_eq!(chat_requests[0].content["messages"][0]["role"], "system"); -async fn decision_handler( - State(state): State, - AxumJson(request): AxumJson, -) -> AxumJson { - state.requests.lock().unwrap().push(request); - AxumJson(state.decision) + let (_, responses_requests) = run_buffered_classifier(WireProtocol::OpenaiResponses).await; + assert!(responses_requests[0].content["instructions"].is_string()); } -async fn decision_server() -> (String, Arc>>) { - decision_server_for(decision()).await -} +#[tokio::test] +async fn buffered_classifier_routes_responses_and_anthropic_inbound_profiles() { + for (inbound, call_name) in [ + (WireProtocol::OpenaiResponses, "openai.responses"), + (WireProtocol::AnthropicMessages, "anthropic.messages"), + ] { + let mut config = classifier_config(false); + config.targets.insert( + "weak".into(), + binding(inbound, &format!("provider/weak-{}", inbound.label()), 1.0), + ); + config.enabled_inbound_profiles = BTreeSet::from([inbound]); + config.default_targets = ProtocolDefaults::default(); + match inbound { + WireProtocol::OpenaiResponses => { + config.default_targets.openai_responses = "weak".into() + } + WireProtocol::AnthropicMessages => { + config.default_targets.anthropic_messages = "weak".into() + } + WireProtocol::OpenaiChat => unreachable!(), + } + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmExecutionNextFn = Arc::new(move |request| { + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + if model == "provider/classifier" { + return Ok(chat_response(&model, CLASSIFIER_VERDICT)); + } + Ok(match inbound { + WireProtocol::OpenaiResponses => { + assert!(request.content["input"].is_array()); + responses_response(&model, "efficient answer") + } + WireProtocol::AnthropicMessages => { + assert!(request.content["messages"].is_array()); + anthropic_response(&model, "efficient answer") + } + WireProtocol::OpenaiChat => unreachable!(), + }) + }) + }); -async fn decision_server_for( - decision: RoutingDecision, -) -> (String, Arc>>) { - let requests = Arc::new(Mutex::new(Vec::new())); - let state = DecisionState { - requests: Arc::clone(&requests), - decision, - }; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new() - .route("/v1/routing/decision", post(decision_handler)) - .with_state(state), - ) - .await - .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), requests) + let response = runtime + .execute_buffered(call_name, request_for(inbound), next) + .await + .unwrap(); + assert!( + response.to_string().contains("efficient answer"), + "classifier response for {inbound:?}: {response}" + ); + } } -async fn health_server(status: StatusCode, body: &'static str) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || async move { (status, [("content-type", "application/json")], body) }), - ), - ) - .await - .unwrap(); +#[tokio::test] +async fn unavailable_classifier_fails_closed_without_retrying_the_judge() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + if model == "provider/classifier" { + return Err(provider_error(UpstreamFailureClass::Connection, None)); + } + Ok(chat_response(&model, "capable answer")) + }) }); - format!("http://{address}/nested/v1/routing/decision?source=test#ignored") -} -async fn recovering_health_server(failures_before_ready: usize) -> (String, Arc) { - let attempts = Arc::new(AtomicUsize::new(0)); - let captured = Arc::clone(&attempts); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve( - listener, - Router::new().route( - "/health", - get(move || { - let attempts = Arc::clone(&captured); - async move { - if attempts.fetch_add(1, Ordering::SeqCst) < failures_before_ready { - ( - StatusCode::SERVICE_UNAVAILABLE, - AxumJson(json!({"status": "starting"})), - ) - } else { - (StatusCode::OK, AxumJson(json!({"status": "ok"}))) - } - } - }), - ), - ) + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) .await .unwrap(); - }); - (format!("http://{address}/v1/routing/decision"), attempts) + assert_eq!( + response["choices"][0]["message"]["content"], + "capable answer" + ); + assert_eq!( + *models.lock().unwrap(), + vec!["provider/classifier", "provider/strong"] + ); } -async fn managed_buffered_events( - runtime: SwitchyardRuntime, - next: LlmExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); +#[tokio::test] +async fn classifier_reconsults_on_retry_without_session_affinity() { + let mut config = classifier_config(false); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let weak_calls = Arc::new(AtomicUsize::new(0)); + let weak_call_count = Arc::clone(&weak_calls); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let weak_call_count = Arc::clone(&weak_call_count); + let captured = Arc::clone(&captured); Box::pin(async move { - runtime - .execute_buffered("openai.chat_completions", request, next) - .await + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + if model == "provider/classifier" { + return Ok(chat_response(&model, CLASSIFIER_VERDICT)); + } + if weak_call_count.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error( + UpstreamFailureClass::RetryableStatus, + Some(503), + )); + } + Ok(chat_response(&model, "retried answer")) }) }); - llm_call_execute( - LlmCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .build(), - ) - .await - .unwrap(); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + assert_eq!( + response["choices"][0]["message"]["content"], + "retried answer" + ); + assert_eq!( + *models.lock().unwrap(), + vec![ + "provider/classifier", + "provider/weak", + "provider/classifier", + "provider/weak" + ] + ); } -async fn managed_stream_events( - runtime: SwitchyardRuntime, - next: LlmStreamExecutionNextFn, -) -> Vec { - let subscriber_name = format!("switchyard-stream-accounting-{}", uuid::Uuid::now_v7()); - let events = Arc::new(Mutex::new(Vec::::new())); - let captured = Arc::clone(&events); - register_subscriber( - &subscriber_name, - Arc::new(move |event| captured.lock().unwrap().push(event.clone())), - ) - .unwrap(); - let runtime = Arc::new(runtime); - let func: LlmStreamExecutionNextFn = Arc::new(move |request| { - let runtime = Arc::clone(&runtime); - let next = Arc::clone(&next); +#[tokio::test] +async fn classifier_session_affinity_skips_repeated_judge_calls() { + let runtime = SwitchyardRuntime::new(classifier_config(true)).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); Box::pin(async move { - runtime - .execute_stream("openai.chat_completions", request, next) - .await + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(model.clone()); + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "answer" + }; + Ok(chat_response(&model, text)) }) }); - let mut stream = llm_stream_call_execute( - LlmStreamCallExecuteParams::builder() - .name("openai.chat_completions") - .request(chat_request()) - .func(func) - .collector(Box::new(|_| Ok(()))) - .finalizer(Box::new(|| json!({"done": true}))) - .build(), - ) - .await - .unwrap(); - while stream.next().await.is_some() {} - drop(stream); - flush_subscribers().unwrap(); - deregister_subscriber(&subscriber_name).unwrap(); - Arc::try_unwrap(events).unwrap().into_inner().unwrap() -} -#[tokio::test] -async fn buffered_accounting_records_only_the_terminal_committed_route() { - let (url, _) = decision_server().await; - let successful = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| Box::pin(async { Ok(chat_response()) })), - ) - .await; - let marks = successful - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].data().unwrap()["applied"], true); + for _ in 0..2 { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next.clone()) + .await + .unwrap(); + } assert_eq!( - marks[0].data().unwrap()["model_transition"]["baseline"]["model"], - "baseline" - ); - let summary = successful - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); - - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") + *models.lock().unwrap(), + vec!["provider/classifier", "provider/weak", "provider/weak"] ); } #[tokio::test] -async fn retry_then_success_records_one_terminal_route_with_matching_mark_identity() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - if seen.fetch_add(1, Ordering::SeqCst) == 0 { - Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry once".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })) - } else { - Ok(chat_response()) - } - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert_eq!(requests[1].attempt.routing_attempt, 2); - drop(requests); - - let start = events - .iter() - .find(|event| { - event.name() == "openai.chat_completions" - && event.scope_category() == Some(ScopeCategory::Start) +async fn buffered_dispatch_uses_relay_binding_and_preserves_same_protocol_json() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let captured = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&captured); + let expected = chat_response("provider/chat", "routed"); + let provider_response = expected.clone(); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + let response = provider_response.clone(); + Box::pin(async move { + capture.lock().unwrap().push(request); + Ok(response) }) - .unwrap(); - let marks = events - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .collect::>(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].parent_uuid(), Some(start.uuid())); - assert_eq!(marks[0].data().unwrap()["payload"]["routing_attempt"], 2); + }); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await .unwrap(); - assert_eq!(summary.contributions.len(), 1); - let contribution = &summary.contributions[0]; - assert!(contribution.applied); - assert_eq!(contribution.payload.as_ref().unwrap()["routing_attempt"], 2); - assert_eq!(marks[0].data().unwrap()["id"], json!(contribution.id)); + assert_eq!(response, expected); + + let requests = captured.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].content["model"], "provider/chat"); + assert_eq!(requests[0].content["stream"], false); assert_eq!( - marks[0].data().unwrap()["sequence"], - json!(contribution.sequence) + requests[0].content["provider_extension"], + json!({"preserve": true}) ); + assert_eq!( + requests[0].headers[INTERNAL_DISPATCH_URL_HEADER], + "https://provider.example.com/v1/chat/completions" + ); + assert!(!requests[0].headers.contains_key("authorization")); + assert!(!requests[0].headers.contains_key("x-nemo-relay-session-id")); } #[tokio::test] -async fn oversized_decision_metadata_limits_accounting_without_failing_provider_success() { - let mut oversized = decision(); - oversized - .metadata - .insert("oversized_evidence".into(), json!("x".repeat(20_000))); - let (url, _) = decision_server_for(oversized).await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(chat_response()) - }) - }), - ) - .await; - - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") +async fn cross_protocol_buffered_route_uses_switchyard_translation_both_ways() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + config.targets.insert( + "anthropic".into(), + binding(WireProtocol::AnthropicMessages, "provider/anthropic", 1.0), ); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmExecutionNextFn = Arc::new(move |request| { + Box::pin(async move { + assert_eq!(request.content["model"], "provider/anthropic"); + assert_eq!(request.content["messages"][0]["role"], "user"); + assert!(request.content.get("input").is_none()); + Ok(json!({ + "id": "msg-test", + "type": "message", + "role": "assistant", + "model": "provider/anthropic", + "content": [{"type": "text", "text": "translated"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 1} + })) }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await .unwrap(); - assert_eq!(summary.status, LlmOptimizationSummaryStatus::Partial); - assert!(summary.contributions.is_empty()); - assert!( - summary - .limitations - .iter() - .any(|limitation| limitation == "contribution_limit_exceeded") - ); + assert_eq!(response["choices"][0]["message"]["content"], "translated"); } #[tokio::test] -async fn malformed_baseline_is_telemetry_only_and_does_not_change_the_selected_route() { - let mut drifted = decision(); - drifted.baseline_route.as_mut().unwrap().target_model = "drifted".into(); - let (url, _) = decision_server_for(drifted).await; - let events = managed_buffered_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|request| { +async fn deterministic_seed_reproduces_the_same_target_sequence() { + fn config() -> SwitchyardConfig { + let mut config = chat_config(); + config.targets.insert( + "other".into(), + binding(WireProtocol::OpenaiChat, "provider/other", 1.0), + ); + config + } + + async fn sequence(runtime: &SwitchyardRuntime) -> Vec { + let captured = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&captured); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); Box::pin(async move { - assert_eq!(request.content["model"], "selected"); - Ok(chat_response()) - }) - }), - ) - .await; - assert!(events.iter().any(|event| { - event.name() == "switchyard.routing.error" - && event.data().is_some_and(|data| { - data["error_class"] == "baseline_binding" - && data["error"] - .as_str() - .is_some_and(|error| error.contains("exact Relay binding")) + let model = request.content["model"].as_str().unwrap().to_string(); + capture.lock().unwrap().push(model.clone()); + Ok(chat_response(&model, "ok")) }) - })); - assert!( - events - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") - ); + }); + for _ in 0..12 { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next.clone()) + .await + .unwrap(); + } + captured.lock().unwrap().clone() + } + + let first = SwitchyardRuntime::new(config()).unwrap(); + let second = SwitchyardRuntime::new(config()).unwrap(); + let first_sequence = sequence(&first).await; + let second_sequence = sequence(&second).await; + assert_eq!(first_sequence, second_sequence); + assert!(first_sequence.contains(&"provider/chat".to_string())); + assert!(first_sequence.contains(&"provider/other".to_string())); } #[tokio::test] -async fn observe_only_accounting_is_visible_but_not_applied() { - let (url, _) = decision_server().await; - let mut observe = config(url); - observe.mode = RoutingMode::ObserveOnly; - let events = managed_buffered_events( - SwitchyardRuntime::new(observe).unwrap(), - Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) - }) - }), - ) - .await; - let contribution = events - .iter() - .find(|event| event.name() == "nemo_relay.llm.optimization") - .and_then(Event::data) - .unwrap(); - assert_eq!(contribution["applied"], false); - let summary = events - .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) +async fn retryable_provider_failure_starts_a_fresh_libsy_run() { + let mut config = chat_config(); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + Box::pin(async move { + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error( + UpstreamFailureClass::RetryableStatus, + Some(503), + )); + } + Ok(chat_response( + request.content["model"].as_str().unwrap(), + "retried", + )) }) + }); + + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(!summary.contributions[0].applied); - assert!(summary.tokens_saved.total_tokens.is_none()); + assert_eq!(response["choices"][0]["message"]["content"], "retried"); + assert_eq!(calls.load(Ordering::SeqCst), 2); } #[tokio::test] -async fn streaming_accounting_commits_on_the_first_successful_item_only() { - let (url, _) = decision_server().await; - let committed = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(|_| { - Box::pin(async { - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - chat_chunk("ok", json!("stop")), - )]))) - }) - }), - ) - .await; - assert_eq!( - committed - .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 +async fn non_retryable_failure_dispatches_trusted_fallback_exactly_once() { + let mut config = chat_config(); + config.targets.insert( + "fallback".into(), + binding(WireProtocol::OpenaiChat, "provider/fallback", 0.0), ); + config.default_targets.openai_chat = "fallback".into(); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let models = Arc::new(Mutex::new(Vec::new())); + let capture = Arc::clone(&models); + let next: LlmExecutionNextFn = Arc::new(move |request| { + let capture = Arc::clone(&capture); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + capture.lock().unwrap().push(model.clone()); + if model == "provider/chat" { + return Err(provider_error( + UpstreamFailureClass::InvalidRequest, + Some(400), + )); + } + Ok(chat_response(&model, "fallback")) + }) + }); - let (url, _) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let fallback = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); - Box::pin(async move { - let items = if seen.fetch_add(1, Ordering::SeqCst) == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - }))] - } else { - vec![Ok(chat_chunk("fallback", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) - }) - }), - ) - .await; - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert!( - fallback - .iter() - .all(|event| event.name() != "nemo_relay.llm.optimization") + let response = runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + .unwrap(); + assert_eq!(response["choices"][0]["message"]["content"], "fallback"); + assert_eq!( + *models.lock().unwrap(), + vec!["provider/chat", "provider/fallback"] ); } #[tokio::test] -async fn committed_stream_error_keeps_one_route_and_never_redispatches() { - let (url, decisions) = decision_server().await; - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let events = managed_stream_events( - SwitchyardRuntime::new(config(url)).unwrap(), - Arc::new(move |_| { - let seen = Arc::clone(&seen); +async fn concurrent_runs_dispatch_independently() { + let runtime = Arc::new(SwitchyardRuntime::new(chat_config()).unwrap()); + let calls = Arc::new(AtomicUsize::new(0)); + let next: LlmExecutionNextFn = { + let calls = Arc::clone(&calls); + Arc::new(move |request| { + let calls = Arc::clone(&calls); Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - Ok(LlmJsonStream::new(futures_stream::iter(vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed after commit".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]))) + calls.fetch_add(1, Ordering::SeqCst); + Ok(chat_response( + request.content["model"].as_str().unwrap(), + "ok", + )) }) - }), - ) - .await; + }) + }; + let runs = (0..16).map(|_| { + let runtime = Arc::clone(&runtime); + let next = next.clone(); + tokio::spawn(async move { + runtime + .execute_buffered("openai.chat_completions", chat_request(), next) + .await + }) + }); + for result in futures_util::future::join_all(runs).await { + assert!(result.unwrap().is_ok()); + } + assert_eq!(calls.load(Ordering::SeqCst), 16); +} - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); - assert_eq!( - events +#[tokio::test] +async fn streaming_classifier_dispatches_streamed_judge_and_routed_calls() { + let runtime = SwitchyardRuntime::new(classifier_config(false)).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); + Box::pin(async move { + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request); + let text = if model == "provider/classifier" { + CLASSIFIER_VERDICT + } else { + "streamed answer" + }; + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + &model, text, + ))]))) + }) + }); + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) + .await + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output .iter() - .filter(|event| event.name() == "nemo_relay.llm.optimization") - .count(), - 1 + .filter_map(|item| item.as_ref().ok()) + .any(|event| event["choices"][0]["delta"]["content"] == "streamed answer") ); - let summary = events + + let requests = requests.lock().unwrap(); + let models = requests .iter() - .find_map(|event| { - event - .annotated_response() - .and_then(|response| response.optimization_summary.as_ref()) - }) - .unwrap(); - assert_eq!(summary.contributions.len(), 1); - assert!(summary.contributions[0].applied); + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); + assert_eq!(models, vec!["provider/classifier", "provider/weak"]); assert!( - summary - .limitations + requests .iter() - .any(|limitation| limitation == "stream_interrupted") + .all(|request| request.content["stream"] == true) ); + assert!(requests[0].content["response_format"].is_object()); } #[tokio::test] -async fn retry_exhaustion_redecides_four_times_then_dispatches_fallback_once() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); +async fn streaming_classifier_supports_an_openai_responses_judge_target() { + let mut config = classifier_config(false); + config.targets.insert( + "classifier".into(), + binding( + WireProtocol::OpenaiResponses, + "provider/classifier-responses", + 1.0, + ), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { + let captured = Arc::clone(&captured); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst) + 1; - if attempt <= 4 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "temporarily unavailable".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - })); + let model = request.content["model"].as_str().unwrap().to_string(); + captured.lock().unwrap().push(request); + if model == "provider/classifier-responses" { + return Ok(LlmJsonStream::new(stream::iter([ + Ok(json!({ + "type": "response.created", + "response": {"id": "resp-test", "model": model} + })), + Ok(json!({ + "type": "response.output_text.delta", + "output_index": 0, + "content_index": 0, + "delta": CLASSIFIER_VERDICT + })), + ]))); } - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + &model, + "streamed answer", + ))]))) }) }); - let response = runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(response["choices"][0]["message"]["content"], "ok"); - assert_eq!(dispatches.load(Ordering::SeqCst), 5); - let requests = decisions.lock().unwrap(); - assert_eq!(requests.len(), 4); - assert_eq!(requests[3].attempt.routing_attempt, 4); + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output + .iter() + .filter_map(|item| item.as_ref().ok()) + .any(|event| event["choices"][0]["delta"]["content"] == "streamed answer") + ); + let requests = requests.lock().unwrap(); + let models = requests + .iter() + .map(|request| request.content["model"].as_str().unwrap()) + .collect::>(); assert_eq!( - requests[3].attempt.previous_route.as_deref(), - Some("selected-chat") + models, + vec!["provider/classifier-responses", "provider/weak"] ); + assert!(requests[0].content["text"]["format"].is_object()); } #[tokio::test] -async fn non_retryable_provider_failure_bypasses_retry_loop() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); +async fn streaming_prefetch_retries_before_commit() { + let mut config = chat_config(); + config.max_retries = 1; + let runtime = SwitchyardRuntime::new(config).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let capture = Arc::clone(&capture); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - if attempt == 0 { - return Err(FlowError::Upstream(UpstreamFailure { - status: Some(401), - body: "unauthorized".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Authentication, - })); + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(provider_error(UpstreamFailureClass::Connection, None)); } - Ok(chat_response()) + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + "provider/chat", + "retried", + ))]))) }) }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 1); + .unwrap() + .collect::>() + .await; + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert!(output.iter().all(Result::is_ok)); } #[tokio::test] -async fn observe_only_records_one_decision_and_dispatches_only_the_trusted_default() { - let (url, decisions) = decision_server().await; - let mut config = config(url); - config.mode = RoutingMode::ObserveOnly; +async fn first_provider_error_event_retries_before_commit() { + let mut config = chat_config(); + config.max_retries = 1; let runtime = SwitchyardRuntime::new(config).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); - let next: LlmExecutionNextFn = Arc::new(move |request| { - let seen = Arc::clone(&seen); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let capture = Arc::clone(&capture); Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - assert_eq!(request.content["model"], "fallback"); - Ok(chat_response()) + if capture.fetch_add(1, Ordering::SeqCst) == 0 { + return Ok(LlmJsonStream::new(stream::iter([Ok(json!({ + "error": {"message": "temporary provider failure"} + }))]))); + } + Ok(LlmJsonStream::new(stream::iter([Ok(chat_chunk( + "provider/chat", + "retried", + ))]))) }) }); - runtime - .execute_buffered("openai.chat_completions", chat_request(), next) + + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); + .unwrap() + .collect::>() + .await; + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert!(output.iter().all(Result::is_ok)); } #[tokio::test] -async fn streaming_retries_before_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); +async fn streaming_late_error_is_propagated_without_retry() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); + let capture = Arc::clone(&capture); Box::pin(async move { - let attempt = seen.fetch_add(1, Ordering::SeqCst); - let items = if attempt == 0 { - vec![Err(FlowError::Upstream(UpstreamFailure { - status: Some(503), - body: "retry".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::RetryableStatus, - }))] - } else { - vec![Ok(chat_chunk("ok", json!("stop")))] - }; - Ok(LlmJsonStream::new(futures_stream::iter(items))) + capture.fetch_add(1, Ordering::SeqCst); + Ok(LlmJsonStream::new(stream::iter([ + Ok(chat_chunk("provider/chat", "partial")), + Err(provider_error(UpstreamFailureClass::Connection, None)), + ]))) }) }); - let stream = runtime + + let output = runtime .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 1); - assert!(output[0].is_ok()); - assert_eq!(dispatches.load(Ordering::SeqCst), 2); - assert_eq!(decisions.lock().unwrap().len(), 2); + .unwrap() + .collect::>() + .await; + assert!(output.first().is_some_and(Result::is_ok)); + assert!(output.last().is_some_and(Result::is_err)); + assert_eq!(calls.load(Ordering::SeqCst), 1); } #[tokio::test] -async fn streaming_never_retries_after_first_item() { - let (url, decisions) = decision_server().await; - let runtime = SwitchyardRuntime::new(config(url)).unwrap(); - let dispatches = Arc::new(AtomicUsize::new(0)); - let seen = Arc::clone(&dispatches); +async fn late_provider_error_event_is_propagated_without_retry() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let capture = Arc::clone(&calls); let next: LlmStreamExecutionNextFn = Arc::new(move |_| { - let seen = Arc::clone(&seen); + let capture = Arc::clone(&capture); Box::pin(async move { - seen.fetch_add(1, Ordering::SeqCst); - let items = vec![ - Ok(chat_chunk("partial", Json::Null)), - Err(FlowError::Upstream(UpstreamFailure { - status: None, - body: "connection closed".into(), - headers: BTreeMap::new(), - class: UpstreamFailureClass::Connection, - })), - ]; - Ok(LlmJsonStream::new(futures_stream::iter(items))) + capture.fetch_add(1, Ordering::SeqCst); + Ok(LlmJsonStream::new(stream::iter([ + Ok(chat_chunk("provider/chat", "partial")), + Ok(json!({"error": {"message": "late provider failure"}})), + ]))) }) }); - let stream = runtime + + let output = runtime .execute_stream("openai.chat_completions", chat_request(), next) .await - .unwrap(); - let output = stream.collect::>().await; - assert_eq!(output.len(), 2); - assert!(output[0].is_ok()); - assert!(output[1].is_err()); - assert_eq!(dispatches.load(Ordering::SeqCst), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); + .unwrap() + .collect::>() + .await; + assert!(output.first().is_some_and(Result::is_ok)); + assert!(output.last().is_some_and(Result::is_err)); + assert_eq!(calls.load(Ordering::SeqCst), 1); } -fn responses_config(decision_api_url: String) -> SwitchyardConfig { - let targets = BTreeMap::from([ - ( - "resp-a".into(), - binding(WireProtocol::OpenaiResponses, "model-a"), - ), - ( - "resp-b".into(), - binding(WireProtocol::OpenaiResponses, "model-b"), - ), - ]); - SwitchyardConfig { - decision_api_url, - decision_profile_id: "stage_router".into(), - request_materialization: RequestMaterialization::SummaryOnly, - context_mode: ContextMode::PayloadOnly, - decision_timeout_millis: 1_000, - targets, - default_targets: ProtocolDefaults { - openai_chat: String::new(), - openai_responses: "resp-a".into(), - anthropic_messages: String::new(), - }, - enabled_inbound_profiles: BTreeSet::from([WireProtocol::OpenaiResponses]), - ..SwitchyardConfig::default() - } -} +#[tokio::test] +async fn cross_protocol_stream_uses_switchyard_translation_both_ways() { + let mut config = chat_config(); + config.targets.get_mut("chat").unwrap().weight = 0.0; + config.targets.insert( + "anthropic".into(), + binding(WireProtocol::AnthropicMessages, "provider/anthropic", 1.0), + ); + let runtime = SwitchyardRuntime::new(config).unwrap(); + let next: LlmStreamExecutionNextFn = Arc::new(move |request| { + Box::pin(async move { + assert_eq!(request.content["model"], "provider/anthropic"); + Ok(LlmJsonStream::new(stream::iter([ + Ok(json!({ + "type": "message_start", + "message": { + "id": "msg-stream", + "type": "message", + "role": "assistant", + "model": "provider/anthropic", + "content": [], + "usage": {"input_tokens": 2, "output_tokens": 0} + } + })), + Ok(json!({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""} + })), + Ok(json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "translated"} + })), + Ok(json!({"type": "content_block_stop", "index": 0})), + Ok(json!({ + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 1} + })), + Ok(json!({"type": "message_stop"})), + ]))) + }) + }); -fn responses_decision() -> RoutingDecision { - RoutingDecision { - route: crate::contract::RoutingTarget { - tier: "efficient".into(), - target_model: "model-b".into(), - backend_id: "resp-b".into(), - target_protocol_profile: "openai_responses".into(), - target_endpoint: "/v1/responses".into(), - }, - baseline_route: None, - ..decision() - } + let output = runtime + .execute_stream("openai.chat_completions", chat_request(), next) + .await + .unwrap() + .collect::>() + .await; + assert!(output.iter().all(Result::is_ok)); + assert!( + output + .iter() + .filter_map(|result| result.as_ref().ok()) + .any(|event| { + event["choices"][0]["delta"]["content"] + .as_str() + .is_some_and(|text| text == "translated") + }) + ); } -// When every configured target shares the inbound protocol, no cross-protocol translation can -// occur, so the portability guard is skipped: a non-portable Responses request (real Codex -// extensions) is routed through the Decision API with its provider-specific fields preserved. #[tokio::test] -async fn same_protocol_targets_route_nonportable_streaming_requests() { - let (url, decisions) = decision_server_for(responses_decision()).await; - let runtime = SwitchyardRuntime::new(responses_config(url)).unwrap(); - let mut nonportable = request(WireProtocol::OpenaiResponses); - nonportable.content["reasoning"] = json!({"effort": "high"}); - nonportable.content["store"] = json!(true); - let next: LlmStreamExecutionNextFn = Arc::new(|request| { - Box::pin(async move { - assert_eq!(request.content["model"], "model-b"); - assert_eq!(request.content["store"], true); - assert_eq!(request.content["reasoning"]["effort"], "high"); - Ok(LlmJsonStream::new(futures_stream::iter(vec![Ok( - json!({"type": "response.output_text.delta", "delta": "ok"}), - )]))) - }) +async fn same_protocol_stream_round_trip_preserves_raw_provider_event() { + let runtime = SwitchyardRuntime::new(chat_config()).unwrap(); + let raw = chat_chunk("provider/chat", "exact"); + let provider_chunk = raw.clone(); + let next: LlmStreamExecutionNextFn = Arc::new(move |_| { + let chunk = provider_chunk.clone(); + Box::pin(async move { Ok(LlmJsonStream::new(stream::iter([Ok(chunk)]))) }) }); + let output = runtime - .execute_stream("openai.responses", nonportable, next) + .execute_stream("openai.chat_completions", chat_request(), next) .await .unwrap() .collect::>() .await; assert_eq!(output.len(), 1); - assert_eq!(decisions.lock().unwrap().len(), 1); + assert_eq!(output[0].as_ref().ok(), Some(&raw)); +} + +#[test] +fn plugin_source_does_not_use_relay_translation_codecs() { + let component = include_str!("../../src/component.rs"); + let translation = include_str!("../../src/translation.rs"); + let stream_translation = include_str!("../../src/stream_translation.rs"); + for source in [component, translation, stream_translation] { + assert!(!source.contains("nemo_relay::codec")); + } } diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index 054e11189..ec52646ae 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -34,8 +34,8 @@ entries in `plugins.toml`. - [PII Redaction](/configure-plugins/pii-redaction/about) sanitizes sensitive data in observability payloads. - [Switchyard (Experimental)](/configure-plugins/switchyard/about) routes - requests using decisions from a separately running Switchyard Decision API - service. + requests in process through Switchyard libsy while Relay dispatches provider + calls. - [Model Pricing](/configure-plugins/model-pricing) configures catalog sources for cost estimates on managed LLM responses. diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx index 321a4d43d..cbf6b42ae 100644 --- a/docs/configure-plugins/switchyard/about.mdx +++ b/docs/configure-plugins/switchyard/about.mdx @@ -1,7 +1,7 @@ --- title: "Switchyard (Experimental)" sidebar-title: "Switchyard (Experimental)" -description: "Set up and validate the experimental Switchyard Decision API integration for NeMo Relay 0.6.0." +description: "Route LLM calls in process with Switchyard libsy." position: 5 --- import { MermaidStyles } from "@/components/MermaidStyles"; @@ -9,196 +9,70 @@ import { MermaidStyles } from "@/components/MermaidStyles"; {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -> **Experimental:** The Switchyard integration is an early-access feature. It -> is not enabled in default Relay builds, its configuration and contracts can -> change, and the current deployment requires a separately running Switchyard -> Decision API service. +> **Experimental:** The Switchyard component is compile-time optional and its +> library contract may change before the compatible Switchyard 0.2 crates are +> published. -> **NeMo Relay 0.6.0 architecture:** This release uses the external Switchyard -> Decision API from the -> [`topic/nemo-relay-integration`](https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration) -> branch. +The Switchyard component runs libsy's weighted random router or LLM classifier +inside NeMo Relay. Relay drives `Algorithm::run_stream`, performs every +provider call, and sends the real response or error back through +`CallLlmRequest::respond`. -Switchyard is an LLM routing decision engine. The `nemo-relay-switchyard` -plugin asks Switchyard which configured target should handle an LLM request. -Relay validates the decision and performs the authenticated provider request. -For example, a routing profile can send a simple prompt to a lower-cost model -and a complex prompt to a more capable model. +There is no `switchyard-server`, Decision API, service health check, Switchyard +LLM client, or dedicated Switchyard ATOF receiver. ## Architecture -The following diagram shows the NeMo Relay 0.6.0 service boundary: - ```mermaid flowchart LR - Agent["Agent"] -->|LLM request| Relay["NeMo Relay"] - Relay -->|routing request| Decision["Switchyard Decision API"] - Decision -->|backend decision| Relay - Relay -->|authenticated provider request| Backend["Selected LLM backend"] - Relay -->|ATOF events| History["Switchyard ATOF accumulator"] - History -. routing history .-> Decision + Agent["Agent request"] --> Decode["switchyard-translation decode"] + Decode --> Libsy["libsy run_stream"] + Libsy --> Call["CallLlm"] + Call --> EncodeTarget["switchyard-translation encode target"] + EncodeTarget --> Relay["Relay provider dispatch"] + Relay --> DecodeResponse["switchyard-translation decode response"] + DecodeResponse --> Respond["CallLlmRequest.respond"] + Respond --> Final["ReturnToAgent"] + Final --> EncodeCaller["switchyard-translation encode caller"] + EncodeCaller --> Agent class Agent grey-lightest; class Relay green-lightest; - class Decision yellow-lightest; - class History yellow-lightest; - class Backend teal-lightest; + class Libsy yellow-lightest; + class Decode,EncodeTarget,DecodeResponse,EncodeCaller teal-lightest; ``` -The components divide responsibility as follows: - -| Component | Responsibilities | -| --- | --- | -| Switchyard | Selects a backend and accumulates ATOF routing history for history-based profiles. | -| NeMo Relay | Owns provider credentials, target bindings, decision validation, protocol translation, dispatch, retries, trusted fallback, and observability. | - -Relay does not start or supervise the Switchyard service. Start Switchyard -before Relay activates the plugin. Relay derives the service's root `/health` -URL from `decision_api_url` and fails activation unless the endpoint returns -`{"status":"ok"}`. +## Responsibility Boundary -## Prerequisites - -Install the following prerequisites before you start: - -| Requirement | Version or Value | +| Owner | Responsibilities | | --- | --- | -| NeMo Relay | Tag `0.6.0` | -| Rust | `1.96.1` | -| Switchyard branch | `topic/nemo-relay-integration` | -| Switchyard commit | `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8` | -| Local commands | `git`, `python3`, and `curl` | - -Keep the repositories in the following sibling layout. The validation script -uses this layout by default: - -```text -/ -├── NeMo-Relay/ -└── Switchyard-topic-nemo-relay-integration/ -``` - -## Build and Run the Compatibility Test +| libsy | Random selection or classifier consultation and tier selection, decision emission, algorithm lifecycle, and consuming each response supplied by Relay. | +| `switchyard-translation` | Decode and encode OpenAI Chat, OpenAI Responses, and Anthropic Messages requests, buffered responses, and stream events. | +| NeMo Relay | Provider bindings, credentials, URL and endpoint selection, physical dispatch, retries, trusted fallback, scopes, and observability. | -Use the following procedure to clone and pin both repositories: +Translation also runs for same-protocol calls because the request and response +must pass through libsy's neutral types. Switchyard's in-memory preservation +replays same-protocol provider fields exactly. When the target protocol differs, +Switchyard translates only the common representable contract and rejects +configured loss. -```bash -mkdir relay-switchyard-0.6 -cd relay-switchyard-0.6 - -git clone --branch 0.6.0 \ - https://github.com/NVIDIA/NeMo-Relay.git \ - NeMo-Relay - -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - Switchyard-topic-nemo-relay-integration - -git -C Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 +## Try the No-Service Example -cd NeMo-Relay -``` - -The Switchyard plugin is excluded from default Relay CLI builds. Build the CLI -with the optional feature explicitly: - -```bash -cargo build -p nemo-relay-cli --features switchyard -``` - -Run the real-service compatibility test from the NeMo Relay repository root: +From a Relay source checkout: ```bash examples/switchyard/run-real-e2e.sh ``` -The script prints the verified Switchyard revision and ends with the following -line when routing succeeds: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -The test starts these local processes: - -| Process | Port | Readiness Check or Purpose | -| --- | --- | --- | -| Switchyard Decision API | `4000` | Serves `/health`, routing decisions, and ATOF ingestion. | -| NeMo Relay | `4041` | Serves `/healthz` and the OpenAI-compatible gateway. | -| Deterministic provider | `4101` | Records the model selected for each provider request. | - -The first request has no accumulated ATOF history and routes to -`provider/weak`. The script then sends session and tool events to Relay. Relay -exports those events to Switchyard's ATOF accumulator, so the next buffered -request and the final streaming request route to `provider/strong`. The test -also validates the streamed response before reporting success. - -## Configuration Walkthrough +The example starts Relay and one fake provider, exercises deterministic routing +between two semantic targets, and validates buffered and streaming fidelity. -The compatibility test uses -[`real-e2e-plugins.toml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-plugins.toml) -and -[`real-e2e-profiles.yaml`](https://github.com/NVIDIA/NeMo-Relay/blob/release/0.6/examples/switchyard/real-e2e-profiles.yaml). -The following settings establish the routing boundary: +The classifier follows the same architecture but produces two provider-call +steps: a classifier consultation followed by the selected weak or strong +target. Both calls use Relay's configured target bindings and +`switchyard-translation`. -| Setting | Purpose | -| --- | --- | -| `decision_api_url` | Points Relay to the separately running Switchyard Decision API. | -| `decision_profile_id` | Selects the Switchyard routing profile. The profile maps its semantic targets to backend IDs. | -| `targets.` | Binds each returned backend ID to a Relay-owned model, protocol, endpoint, base URL, and credentials. | -| `mode = "enforce"` | Applies a valid Switchyard decision and dispatches to the selected target. | -| `mode = "observe_only"` | Records the hypothetical decision but dispatches once to the trusted same-protocol default. | -| `default_targets` | Defines trusted same-protocol fallbacks for unavailable or invalid decisions and provider failures. | -| `context_mode = "atof_required"` | Requires stable identity and accumulated ATOF history for the selected profile. | -| `atof_endpoint_name` | Selects exactly one named Relay ATOF HTTP stream sink that sends events to Switchyard. | - -For history-based routing, a local ATOF JSONL file is not sufficient. -Switchyard must receive events through the named HTTP sink at -`/v1/atof/events` so its accumulator can provide the history used by the -routing profile. Relay rejects a missing, duplicate, disabled, or invalid -named sink during startup validation. - -For the complete option reference, refer to -[Switchyard Configuration](./configuration.mdx). - -## Troubleshooting - -Use the following table to diagnose common compatibility-test failures: - -| Symptom | Cause and Resolution | -| --- | --- | -| `Switchyard worktree not found: ...` | Place the Switchyard checkout next to `NeMo-Relay` as `Switchyard-topic-nemo-relay-integration`, or set `SWITCHYARD_ROOT` to its absolute path. | -| `Switchyard checkout mismatch: expected ..., found ...` | Check out commit `8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8`. Override `SWITCHYARD_EXPECTED_COMMIT` only when deliberately testing a different contract. | -| A build uses Rust `1.96.0` or reports an unsupported toolchain | Run `rustc --version`, install Rust `1.96.1`, and allow the repository's `rust-toolchain.toml` to select it. | -| `timed out waiting for http://127.0.0.1:4000/health` | Review `switchyard.log`. Confirm the pinned checkout builds and port `4000` is free. | -| `timed out waiting for http://127.0.0.1:4041/healthz` or `process ... exited before http://127.0.0.1:4041/healthz became ready` | Review `relay.log`. Confirm the CLI was built with `--features switchyard`, the Switchyard health check passed, and port `4041` is free. | -| The provider cannot start or requests fail | Review `upstream.log` and confirm port `4101` is free. | -| `E2E logs preserved in ...` | The script preserves its temporary directory after a runtime failure and prints the exact path. Inspect `switchyard.log`, `relay.log`, `upstream.log`, and the captured request outputs there. Successful runs remove the directory. | - -## Capabilities - -The integration provides the following capabilities: - -- `enforce` and `observe_only` routing modes. -- OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages inbound - profiles. -- Buffered and streaming protocol translation through Switchyard's - `switchyard-translation` library. -- Exact Relay-owned backend bindings and per-protocol trusted fallbacks. -- Bounded retries before the first streaming item. -- Canonical routing marks and shared LLM optimization accounting. - -## Pages - -For more information, refer to the following pages: - -- [Switchyard Configuration](./configuration.mdx) - provides the complete option and deployment reference. Review its - [experimental limitations](./configuration.mdx#experimental-limitations) - before adopting the integration. -- [Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard) - contains the versioned configuration and validation scripts used by this - guide. +Continue with [Switchyard Configuration](./configuration.mdx) for the complete +version-2 schema. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx index 54bce55c0..531b28600 100644 --- a/docs/configure-plugins/switchyard/configuration.mdx +++ b/docs/configure-plugins/switchyard/configuration.mdx @@ -1,48 +1,17 @@ --- title: "Switchyard Configuration" sidebar-title: "Configuration" -description: "Configure the experimental Relay-native Switchyard Decision API plugin." +description: "Configure the library-only Relay-native Switchyard plugin." position: 2 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -> **Experimental NeMo Relay 0.6.0 integration:** This release calls a separately -> running Switchyard Decision API. - -Start with the -[Switchyard 0.6.0 setup and validation guide](./about.mdx) -before using this page as the complete configuration reference. - -The Switchyard component connects Relay's CLI gateway to a separately running -[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API. Relay -owns provider credentials, backend endpoints, dispatch, retries, and trusted -fallbacks. The Switchyard service selects a backend. Relay validates the -decision, translates requests and responses in process through the -`switchyard-translation` library, and dispatches directly to the selected -Relay-owned target. - -## How the Configuration Fits Together - -Configure the integration in four core steps. Profiles that use ATOF history -add the fifth requirement: - -1. Set `decision_api_url` and authentication so Relay can call the Switchyard - Decision API. -2. Select the Switchyard routing profile with `decision_profile_id`. -3. Add a Relay-owned `targets` binding for every Switchyard `backend_id` that - the profile can return. Relay validates the returned backend ID, model, - protocol, and endpoint exactly before dispatch. -4. Choose a trusted same-protocol target for each inbound protocol under - `default_targets`. Relay uses these defaults for observe-only traffic and - fail-open dispatch. -5. Configure profiles that depend on ATOF history, including StageRouter, with - `context_mode = "atof_required"` and `atof_endpoint_name`, then configure the - matching observability stream sink. Relay fails startup validation if the - named sink is missing or invalid. - -The following is a complete component configuration. The referenced -environment variables must be set before Relay starts. +The Switchyard component runs libsy in process. It currently supports the +weighted random router and LLM classifier in enforce mode for OpenAI Chat, +OpenAI Responses, and Anthropic Messages managed LLM calls. + +## Complete Example ```toml version = 1 @@ -52,307 +21,174 @@ kind = "switchyard" enabled = true [components.config] -version = 1 -mode = "enforce" +version = 2 priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] +enabled_inbound_profiles = [ + "openai_chat", + "openai_responses", + "anthropic_messages", +] -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] -openai_chat = "openai-default" +openai_chat = "chat-default" openai_responses = "responses-default" anthropic_messages = "anthropic-default" -[components.config.targets.openai-default] -model = "provider/efficient-chat" +[components.config.targets.fast] +model = "provider/fast" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://provider.example.com" +weight = 1 + +[components.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" -[components.config.targets.openai-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" +[components.config.targets.chat-default] +model = "provider/chat-fallback" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 0 [components.config.targets.responses-default] -model = "provider/efficient-responses" +model = "provider/responses-fallback" protocol = "openai_responses" endpoint = "/v1/responses" base_url = "https://provider.example.com" - -[components.config.targets.responses-default.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" +weight = 0 [components.config.targets.anthropic-default] -model = "provider/efficient-messages" +model = "provider/anthropic-fallback" protocol = "anthropic_messages" endpoint = "/v1/messages" base_url = "https://provider.example.com" - -[components.config.targets.anthropic-default.header_env] -x-api-key = "ANTHROPIC_PROVIDER_API_KEY" - -[components.config.targets.capable-chat] -model = "provider/capable-chat" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" - -[components.config.targets.capable-chat.header_env] -authorization = "OPENAI_PROVIDER_AUTHORIZATION" - -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 2 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +weight = 0 ``` -## Decision API and Rollout Options - -The following table describes the Decision API and rollout settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `version` | `1` | Component configuration version. | -| `mode` | `"enforce"` | `enforce` applies valid decisions. `observe_only` records the hypothetical decision but dispatches the trusted same-protocol default once. | -| `priority` | `0` | Component execution priority. | -| `decision_api_url` | Required | HTTP or HTTPS URL for the Switchyard Decision API. | -| `decision_profile_id` | Required | Non-empty Switchyard profile identifier sent with each routing request. | -| `decision_timeout_millis` | `25` | Decision API timeout in milliseconds; must be greater than zero. | -| `decision_headers` | Empty | Optional non-sensitive static Decision API headers. | -| `decision_header_env` | Empty | Map of header names to environment-variable names for sensitive Decision API headers. Referenced values must be present and non-blank. | - -Do not define the same header, including case variants, in both a static header -map and its environment map. Keep bearer tokens, API keys, cookies, and other -secrets in environment variables rather than tracked configuration. - -## Request Materialization - -Every Decision API request contains routing identity, inbound protocol, a -request summary, and the attempt number. `request_materialization` controls how -much additional request content is included. - -| Mode | Material sent to Switchyard | -| --- | --- | -| `none` | No `current_request`; use identity, protocol, summary, and attempt metadata only. | -| `summary_only` | No `current_request`; explicitly selects summary-based routing. | -| `latest_user_prompt` | A provider-valid body containing the latest user material plus an explicit prompt field. The request fails open if no user message is available. | -| `recent_message_window` | The most recent normalized messages, bounded by `recent_message_count`, re-encoded for the inbound protocol. | -| `annotated_request` | The final intercepted provider body plus Relay's normalized request representation. | -| `full_body` | The final intercepted provider request body. | - -Choose the least materialization needed by the selected profile. In particular, -`none` and `summary_only` intentionally share the same current wire behavior but -express different configuration intent. - -## Context, Retries, and Inbound Protocols - -The following table describes context, retry, and inbound protocol settings: - -| Option | Default | Description and constraints | -| --- | --- | --- | -| `context_mode` | Required | `payload_only` or `atof_required`. See below. | -| `max_retries` | `3` | Provider retries after the initial attempt; maximum `10`. Each retry obtains a new Switchyard decision. | -| `recent_message_count` | `8` | Message limit for `recent_message_window`; must be greater than zero. | -| `enabled_inbound_profiles` | All three supported protocols | Non-empty list containing `openai_chat`, `openai_responses`, and/or `anthropic_messages`. | -| `atof_endpoint_name` | Required for `atof_required` | Name of exactly one stream sink under the observability component's `atof.sinks` list. | - -`payload_only` supports request-aware routing without an ATOF dependency. -Relay can create request-scoped synthetic identity when stable identity is not -available. - -`atof_required` is for profiles that classify from accumulated ATOF context. It -requires stable session and request identity plus an enabled, authenticated -Relay HTTP ATOF stream sink whose `name` matches `atof_endpoint_name`. The sink -specifies the Switchyard ingestion URL, must use `transport = "http_post"`, and -must set `field_name_policy = "preserve"`. Relay rejects missing or duplicate -named sinks during startup validation. A local ATOF file sink -does not satisfy this delivery requirement by itself. - -## Target Bindings and Trusted Defaults - -The following table describes target bindings and trusted defaults: +## Component Fields -| Option | Required fields | Purpose | +| Field | Default | Description | | --- | --- | --- | -| `targets.` | `model`, `protocol`, `endpoint`, `base_url` | Binds an exact Switchyard backend ID to a Relay-owned provider destination. | -| `targets..headers` | None | Optional non-sensitive static provider headers. | -| `targets..header_env` | None | Optional provider header names mapped to environment-variable names. Referenced values must be present and non-blank. | -| `default_targets.openai_chat` | Target using `openai_chat` | Trusted fallback for OpenAI Chat Completions requests. | -| `default_targets.openai_responses` | Target using `openai_responses` | Trusted fallback for OpenAI Responses requests. | -| `default_targets.anthropic_messages` | Target using `anthropic_messages` | Trusted fallback for Anthropic Messages requests. | +| `version` | `2` | Library-only schema version. Version 1 is rejected because it used the removed Decision API service contract. | +| `priority` | `0` | Relay LLM execution-intercept priority. | +| `max_retries` | `3` | Provider retries after the initial libsy run. Maximum: `10`. | +| `enabled_inbound_profiles` | All three protocols | Managed Relay profiles intercepted by this component. | +| `algorithm` | Random without a seed | libsy algorithm configuration. Supports `kind = "random"` and `kind = "llm_classifier"`. | +| `targets` | Required | Semantic target names mapped to Relay-owned provider bindings. | +| `default_targets` | Required per enabled profile | Same-protocol targets used for trusted fallback. | -Do not define the same target header, including case variants, in both -`headers` and `header_env`. +### Random Algorithm -Protocol and endpoint pairs are fixed: +`seed` is optional. Configure it for repeatable QA sequences. Each target's +nonnegative `weight` participates in libsy's weighted selection; `0` excludes a +target from normal routing while retaining it as a trusted fallback. -| Protocol | Endpoint | -| --- | --- | -| `openai_chat` | `/v1/chat/completions` | -| `openai_responses` | `/v1/responses` | -| `anthropic_messages` | `/v1/messages` | +### LLM Classifier Algorithm -Each protocol in `enabled_inbound_profiles` requires a corresponding -`default_targets` entry. Defaults for disabled protocols can be omitted. Each -configured default must refer to a target with the matching protocol. Duplicate -exact bindings for model, protocol, endpoint, and base URL are rejected. - -## Recommended Recipes - -### Observe-Only Rollout - -Use observe-only mode to verify decisions and routing marks without sending -traffic to the selected target: - -```toml -[components.config] -mode = "observe_only" -context_mode = "payload_only" -request_materialization = "summary_only" -``` - -Relay calls the Decision API and records the hypothetical selection, but sends -the request to the inbound protocol's trusted default once. Routing retries are -not applied. - -### Payload-Only Request-Aware Routing - -Use recent request content for routing without operating the ATOF accumulator: - -```toml -[components.config] -mode = "enforce" -context_mode = "payload_only" -request_materialization = "recent_message_window" -recent_message_count = 8 -``` - -### ATOF-Backed StageRouter Routing - -Use ATOF context when StageRouter should classify from accumulated session -signals: +The classifier consults one model, evaluates its structured verdict, and then +routes the original request to either the weak or strong target: ```toml -[components.config] -mode = "enforce" -decision_profile_id = "stage-router" -context_mode = "atof_required" -request_materialization = "summary_only" -atof_endpoint_name = "switchyard" -``` - -Configure the matching observability component with a named stream sink: - -```toml -[[components]] -kind = "observability" -enabled = true - -[components.config] -version = 2 - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 +capability_elevated_floor = 0.8 +session_affinity = false +message_hash_fallback = false ``` -## Runtime and Fail-Open Behavior - -The integration handles routing and failures as follows: - -- Relay validates the decision's backend ID, model, protocol, and endpoint - against its target binding before dispatch. Unknown backends, malformed - decisions, and target drift fail open. -- Decision API errors are not recursively retried. Relay immediately dispatches - the trusted same-protocol default. -- Retryable provider connection, timeout, status, context-window, and - model-unavailable failures can obtain a new decision, bounded by - `max_retries`. -- Non-retryable provider failures use the trusted fallback immediately. After - retry exhaustion, Relay dispatches that fallback once. -- A streaming request can retry only before receiving its first upstream item. - After the stream commits, Relay propagates later failures without another - dispatch. -- Relay forwards provider-specific extensions unchanged when every configured - target uses the inbound protocol. If any target uses another protocol, - nonportable extensions fail open to the trusted same-protocol default before - the Decision API call. Request translation failures before provider dispatch - and buffered response translation failures also fail open. After a streaming - response emits its first item, later provider or translation failures - propagate without another dispatch. -- Routing-mark delivery is best-effort and does not change provider results. - -## Experimental Limitations - -Consider the following experimental limitations before adopting the plugin: - -- The plugin is published as `nemo-relay-switchyard` but is excluded from - default Relay CLI builds. Build the CLI with - `cargo build -p nemo-relay-cli --features switchyard`. -- The routing boundary is currently service-based. Relay calls Switchyard's - HTTP Decision API but does not start or supervise the service. Relay derives - `/health` from `decision_api_url` and fails plugin activation unless the - service returns `{"status":"ok"}`. Decision API failures after activation fail - open to the trusted same-protocol default. -- ATOF-backed profiles additionally depend on Switchyard's HTTP ingestion and - accumulator runtime, stable request and session identity, and one named, - authenticated Relay ATOF HTTP stream sink. A local file sink does not populate - the Switchyard accumulator. -- Provider-protocol translation runs in Relay's process through the - `switchyard-translation` Rust library. Routing decisions and ATOF accumulation - remain out of process. -- The Rust Switchyard component is currently registered only by Relay's CLI - gateway. Hermes native plugins and the LangChain, LangGraph, and Deep Agents - integrations do not load it in process. They receive Switchyard routing only - when their provider traffic is explicitly sent through the Relay gateway. -- The Switchyard examples are manual compatibility and trajectory workflows, - not production deployment orchestration. -- Compatibility is pinned to Switchyard commit - [`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8) - on `topic/nemo-relay-integration`. Testing another revision is deliberate and - can expose contract drift. -- The Decision API and service/library boundary are experimental. -- Nonportable provider extensions in configurations that can translate across - protocols, malformed decisions, service failures, pre-dispatch translation - failures, buffered-response translation failures, and target drift fail open - to Relay's trusted same-protocol default. Failures after a streaming response - commits propagate without another dispatch. -- Secrets must remain environment-referenced. The examples generate ephemeral - local credentials and are not credential-management guidance. - -Run `nemo-relay doctor` to validate component configuration before startup. For -the pinned local service and manual compatibility workflow, refer to the -[Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/release/0.6/examples/switchyard). +| Field | Default | Description | +| --- | --- | --- | +| `classifier_target` | Required | Semantic target for the classifier consultation. Must use `openai_chat` or `openai_responses` under the pinned lossless translation contract. | +| `weak_target` | Required | Efficient target selected when the verdict meets the configured thresholds. | +| `strong_target` | Required | Capable target selected for invalid, abstained, unavailable, or below-threshold verdicts. | +| `base_threshold` | Required | Lowest classifier `p_solve` that selects the weak target. Range: `0.0`–`1.0`. | +| `min_confidence` | `0.0` | Lowest classifier confidence that permits weak-target routing. | +| `capability_elevated_floor` | None | Higher `p_solve` threshold for uncertain, unsupported, or unmatched capability verdicts. | +| `session_affinity` | `false` | Reuse the first decision for later requests carrying the same session identity. | +| `message_hash_fallback` | `false` | Use the first user-message hash when no session identity is available. Requires `session_affinity`. | + +Both the classifier consultation and the selected target call are ordinary +`CallLlm` steps fulfilled by Relay. The classifier target uses Relay's normal +binding, authentication, dispatch, and Switchyard translation path; the plugin +does not use `switchyard-llm-client`. + +### Target Bindings + +| Field | Required | Description | +| --- | --- | --- | +| `model` | Yes | Physical provider model. | +| `protocol` | Yes | `openai_chat`, `openai_responses`, or `anthropic_messages`. | +| `endpoint` | Yes | Must match the protocol's standard endpoint. | +| `base_url` | Yes | HTTP or HTTPS provider base URL. | +| `weight` | No | Relative random-routing weight; defaults to `1`. | +| `headers` | No | Static non-sensitive provider headers. | +| `header_env` | No | Header names mapped to environment variables, intended for credentials. | + +Caller credentials are removed before an explicit target dispatch and replaced +with the target's configured credentials from `header_env`. Relay identity +headers are mapped only into libsy `Context` and `Metadata`; the plugin does not +expose them to the algorithm as provider HTTP headers. + +## Translation Rules + +`switchyard-translation` is the component's only wire-format translation +engine. The component does not invoke NeMo Relay codecs for provider conversion. + +- Inbound provider JSON is decoded into Switchyard's neutral request before + `run_stream`. +- Every `CallLlm` request is encoded for the selected target protocol. +- Provider responses and stream events are decoded before + `CallLlmRequest::respond`. +- `ReturnToAgent` is encoded into the caller's original protocol. +- Same-protocol request bodies, buffered responses, and stream events preserve + provider-specific JSON through Switchyard's in-memory preservation contract. +- Cross-protocol routes reject lossy conversions rather than silently removing + a required feature. + +## Retry and Fallback + +Every retry starts a new libsy run. Random routing obtains a fresh selection. +The classifier runs again unless session affinity already retained the routing +decision. Relay returns the actual provider error to libsy before classifying +the original typed Relay error for retry. A failed classifier consultation is +handled by libsy's fail-closed policy and routes to the strong target; it does +not cause Relay to retry the judge. + +Streaming retries are allowed only before the first provider event. Once the +first event commits the stream, later errors are delivered to the caller and do +not trigger another provider call. + +When attempts are exhausted or a non-retryable routing or translation error +occurs, Relay dispatches the configured same-protocol fallback exactly once. + +## Observability + +Relay emits `switchyard.routing.requested`, `switchyard.routing.call`, +`switchyard.routing.decision`, `switchyard.routing.retry`, +`switchyard.routing.error`, and `switchyard.routing.fallback` mark events from +real libsy lifecycle data. The `algorithm` field uses libsy's algorithm name. +Classifier call marks set `is_routed_call = false`; the selected weak or strong +call and the final decision set it to `true`. + +## Current Limits + +- No decision-only or observe-only mode. +- No stage router or ensemble-specific concurrent dispatch. +- No Switchyard-owned provider client, service, or ATOF ingestion. +- No synthesized decision IDs, confidence, baselines, or optimization + contributions. +- Streaming traffic requires the configured classifier target to support + streamed structured-output responses. +- Provider features that cannot be represented losslessly across protocols are + rejected. diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md index 87a2cff68..3a31f5d2f 100644 --- a/examples/switchyard/README.md +++ b/examples/switchyard/README.md @@ -3,112 +3,45 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Switchyard Integration Examples +# Switchyard libsy example -These examples exercise the experimental NeMo Relay 0.6.0 integration with a separately running -Switchyard Decision API service and the in-process Switchyard translation library. They are -manual, local validation workflows rather than production startup orchestration. +This example runs Switchyard routing inside NeMo Relay. It starts only: -For the canonical architecture, setup, configuration, validation, and troubleshooting workflow, -refer to the -[Switchyard 0.6.0 setup and validation guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about). +1. a deterministic local provider; +2. a Relay CLI process built with the `switchyard` feature. -## Required Switchyard Revision +There is no `switchyard-server`, Decision API, health check, or Switchyard ATOF +ingestion service. -The NeMo Relay 0.6.0 scripts require the following public topic branch and commit: - -```text -https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration -8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Clone the Switchyard repository next to the Relay checkout, then pin the -required commit: - -```bash -git clone --branch topic/nemo-relay-integration \ - https://github.com/NVIDIA-NeMo/Switchyard.git \ - ../Switchyard-topic-nemo-relay-integration -git -C ../Switchyard-topic-nemo-relay-integration checkout --detach \ - 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 -``` - -Every real-service script verifies this commit before launching `switchyard-server`. To test a -deliberately different checkout, set both variables explicitly: - -```bash -SWITCHYARD_ROOT=/path/to/Switchyard \ -SWITCHYARD_EXPECTED_COMMIT= \ - examples/switchyard/run-real-e2e.sh -``` - -## Examples - -Run these commands from the root of the NeMo Relay checkout. - -The Relay CLI's Switchyard support is compile-time optional. The scripts enable the `switchyard` -feature automatically; custom builds must pass `--features switchyard`. - -### Manual Switchyard Compatibility Smoke Test - -`run-real-e2e.sh` is a manual compatibility smoke test. It starts the pinned Switchyard server, -Relay, and a fake provider, then verifies cold and warm StageRouter decisions, buffered routing, -SSE routing, and the selected model sequence. It is intended to catch cross-repository service -contract regressions; the CI-safe Relay process regression test covers the faster local behavior -checks. The script requires Rust tooling, Python, and `curl`; its temporary logs are removed after -a successful run. +From the Relay repository root: ```bash examples/switchyard/run-real-e2e.sh +examples/switchyard/run-classifier-e2e.sh ``` -A successful run ends with: - -```text -real Switchyard E2E passed: ['provider/weak', 'provider/strong', 'provider/strong'] -``` - -### Hermes and Ollama Trajectory - -`run-hermes-ollama-smoke.sh` runs a fixed multi-query trajectory through Hermes, Relay, Ollama, -and Switchyard. It requires Docker, Hermes, and the configured local Ollama models. The script -produces ATOF, ATIF, and OTEL artifacts and can leave Phoenix running with -`SWITCHYARD_KEEP_PHOENIX=1`. - -```bash -examples/switchyard/run-hermes-ollama-smoke.sh -``` - -## Configuration Files - -The directory includes the following configuration and support files: - -- `plugins.toml`: minimal plugin configuration example. -- `real-e2e-plugins.toml` and `real-e2e-profiles.yaml`: deterministic fake-provider E2E. -- `hermes-ollama-plugins.toml` and `hermes-ollama-profiles.yaml`: local Ollama trajectory. -- `fake_upstream.py`: deterministic provider used by the service E2E. -- `otel-collector.yaml`: local OTEL artifact export configuration. - -## Runtime Model - -The scripts launch Switchyard as a separate local process on port `4000`. Relay sends routing -requests to `/v1/routing/decision` and, for ATOF-backed profiles, sends events to -`/v1/atof/events`. Relay owns provider credentials, target bindings, dispatch, retries, and -fallback behavior. Relay executes provider-protocol translation in process through Switchyard's -translation library; the Switchyard service owns ATOF accumulation and routing decisions. The -Switchyard component selects its HTTP ingestion destination by the observability stream sink name, -not by duplicating the sink URL. - -The service is not started automatically by Relay outside these examples. A production deployment -must start a compatible Switchyard service before Relay activates the plugin and configure the -Relay plugin with its Decision API URL. Relay derives the service's root `/health` URL from that -configuration and refuses activation unless it reports `{"status":"ok"}`. - -## Artifacts and Troubleshooting - -Trajectory scripts write to `artifacts/` by default. Set `SWITCHYARD_TRAJECTORY_DIR` to choose a -shareable output directory. On failure, logs are preserved and include the verified Switchyard -revision. Do not place API keys or bearer tokens in configuration files; use environment variables -or an untracked secrets file. For service readiness failures, port conflicts, and the exact -temporary log locations used by `run-real-e2e.sh`, refer to the -[canonical Switchyard 0.6.0 guide](https://docs.nvidia.com/nemo/relay/v0.6.0/configure-plugins/switchyard/about#troubleshooting). +The first script sends buffered and streaming OpenAI Chat requests through a +seeded, weighted libsy random router. It verifies that both configured provider +models are selected, that Relay performs every physical request, and that a +provider-specific `system_fingerprint` survives the same-protocol stream round +trip. The second script verifies the classifier consultation, weak-tier +selection, structured response format, and buffered and streaming final calls. + +The files are: + +- `plugins.toml`: version-2 weighted-random configuration; +- `classifier-plugins.toml`: version-2 LLM-classifier configuration using the + same Relay-owned provider bindings; +- `fake_upstream.py`: deterministic OpenAI-compatible provider; +- `e2e-common.sh`: process and readiness helpers; +- `run-real-e2e.sh`: weighted-random no-service smoke test; +- `run-classifier-e2e.sh`: classifier no-service smoke test. + +The example configurations contain no credentials. For real providers, use +each target's `header_env` map and keep secret values outside tracked +configuration. + +The classifier configuration demonstrates the two-call `run_stream` lifecycle: +the fake `provider/classifier` returns a structured verdict, then libsy selects +`provider/fast`. Both calls are made by Relay; no Switchyard client or service +is involved. diff --git a/examples/switchyard/classifier-plugins.toml b/examples/switchyard/classifier-plugins.toml new file mode 100644 index 000000000..7f932994a --- /dev/null +++ b/examples/switchyard/classifier-plugins.toml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +version = 2 +priority = 0 +max_retries = 2 +enabled_inbound_profiles = ["openai_chat"] + +[components.config.algorithm] +kind = "llm_classifier" +classifier_target = "classifier" +weak_target = "fast" +strong_target = "quality" +base_threshold = 0.5 +min_confidence = 0.5 + +[components.config.default_targets] +openai_chat = "quality" + +[components.config.targets.classifier] +model = "provider/classifier" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" + +[components.config.targets.fast] +model = "provider/fast" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" + +[components.config.targets.quality] +model = "provider/quality" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "http://127.0.0.1:4101" diff --git a/examples/switchyard/e2e-common.sh b/examples/switchyard/e2e-common.sh index bf38627d6..f9687aeda 100755 --- a/examples/switchyard/e2e-common.sh +++ b/examples/switchyard/e2e-common.sh @@ -50,29 +50,6 @@ e2e_wait_for() { return 1 } -e2e_random_token() { - python3 -c 'import secrets; print(secrets.token_hex(24))' -} - -# The experimental Relay integration is tested against a specific public -# Switchyard topic-branch commit. Override this only when intentionally testing -# another revision. -e2e_verify_switchyard_checkout() { - local root="$1" - local expected="${2:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" - local actual - actual="$(git -C "$root" rev-parse HEAD 2>/dev/null)" || { - echo "Switchyard worktree is not a git checkout: $root" >&2 - return 1 - } - if [[ "$actual" != "$expected" ]]; then - echo "Switchyard checkout mismatch: expected $expected, found $actual" >&2 - echo "Set SWITCHYARD_EXPECTED_COMMIT only when intentionally testing another revision." >&2 - return 1 - fi - printf 'Switchyard revision: %s (%s)\n' "$actual" "$(git -C "$root" show -s --format=%s HEAD)" -} - e2e_tail_logs() { local directory="$1" local log diff --git a/examples/switchyard/fake_upstream.py b/examples/switchyard/fake_upstream.py index f27527d14..bc5b5e1d6 100644 --- a/examples/switchyard/fake_upstream.py +++ b/examples/switchyard/fake_upstream.py @@ -8,6 +8,19 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +CLASSIFIER_VERDICT = json.dumps( + { + "recommended_route": "efficient", + "p_solve": 0.9, + "confidence": 0.95, + "abstain": False, + "capability_boundary": "supported", + "primary_rule": "SUP-1", + "crux": "bounded task", + }, + separators=(",", ":"), +) + class Handler(BaseHTTPRequestHandler): log_path: Path @@ -18,16 +31,18 @@ def do_POST(self) -> None: # noqa: N802 with self.log_path.open("a", encoding="utf-8") as output: output.write(json.dumps({"path": self.path, "body": body}) + "\n") model = body.get("model", "fake-model") + content = CLASSIFIER_VERDICT if model == "provider/classifier" else "fake" if body.get("stream"): chunks = [ { "id": "chatcmpl-fake", "object": "chat.completion.chunk", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [ { "index": 0, - "delta": {"role": "assistant", "content": "fake"}, + "delta": {"role": "assistant", "content": content}, "finish_reason": None, } ], @@ -36,6 +51,7 @@ def do_POST(self) -> None: # noqa: N802 "id": "chatcmpl-fake", "object": "chat.completion.chunk", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ] @@ -52,10 +68,11 @@ def do_POST(self) -> None: # noqa: N802 "id": "chatcmpl-fake", "object": "chat.completion", "model": model, + "system_fingerprint": "fp_switchyard_example", "choices": [ { "index": 0, - "message": {"role": "assistant", "content": "fake"}, + "message": {"role": "assistant", "content": content}, "finish_reason": "stop", } ], diff --git a/examples/switchyard/hermes-ollama-plugins.toml b/examples/switchyard/hermes-ollama-plugins.toml deleted file mode 100644 index f562e2c72..000000000 --- a/examples/switchyard/hermes-ollama-plugins.toml +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "hermes-ollama-stage-router" -request_materialization = "full_body" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 120000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "ollama-weak" -openai_responses = "ollama-responses" -anthropic_messages = "ollama-anthropic" - -[components.config.targets.ollama-weak] -model = "llama3.2:latest" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-strong] -model = "qwen3.6:35b" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-responses] -model = "llama3.2:latest" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:11434" - -[components.config.targets.ollama-anthropic] -model = "llama3.2:latest" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:11434" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" -output_directory = "." -filename = "trajectory.atof.jsonl" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.atif] -enabled = true -agent_name = "Hermes" -agent_version = "switchyard-stage-router-smoke" -model_name = "switchyard-stage-router" -output_directory = "." -filename_template = "trajectory-{session_id}.atif.json" - -[components.config.opentelemetry] -enabled = true -transport = "http_binary" -endpoint = "http://127.0.0.1:4318/v1/traces" -service_name = "nemo-relay-switchyard-e2e" -service_namespace = "switchyard-integration" -instrumentation_scope = "nemo-relay-switchyard" -timeout_millis = 5000 - -[components.config.opentelemetry.resource_attributes] -"deployment.environment.name" = "local-e2e" -"switchyard.profile.id" = "hermes-ollama-stage-router" diff --git a/examples/switchyard/hermes-ollama-profiles.yaml b/examples/switchyard/hermes-ollama-profiles.yaml deleted file mode 100644 index 751201851..000000000 --- a/examples/switchyard/hermes-ollama-profiles.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - ollama-weak: - model: llama3.2:latest - format: openai - base_url: http://127.0.0.1:11434/v1 - ollama-strong: - model: qwen3.6:35b - format: openai - base_url: http://127.0.0.1:11434/v1 - -profiles: - hermes-ollama-stage-router: - type: stage_router - capable: ollama-strong - efficient: ollama-weak - fallback_target_on_evict: ollama-strong - picker: efficient_first - # Force the classifier path for ordinary prompts so this smoke demonstrates - # request-complexity routing rather than relying on incidental tool signals. - confidence_threshold: 1.0 - # Each fixed trajectory transition is driven by the latest canonical tool - # result so older critical signals do not leak into the simple follow-up. - signal_recent_window: 1 - classifier: - # Keep classification small and deterministic; qwen3.6:35b remains the - # capable backend selected for the complex trajectory turn. - model: llama3.2:latest - api_key: ${OLLAMA_CLASSIFIER_API_KEY} - base_url: http://127.0.0.1:11434/v1 - timeout_secs: 120.0 - recent_turn_window: 1 - max_tokens: 128 - system_prompt: >- - Select exactly one model tier for the latest user request. Return only - JSON in the form {"tier":"efficient"} or {"tier":"capable"}. Choose efficient - for arithmetic, extraction, formatting, concise factual responses, or - other direct low-risk work. Choose capable for architecture, concurrency, - debugging, multi-step reasoning, subtle correctness constraints, or - requests requiring a defensible technical analysis. In the supplied - State line, severity at least 1.0 always means capable; tests_passed=true - with severity 0.0 means efficient. Classify the latest state even when earlier - conversation turns were more complex. diff --git a/examples/switchyard/otel-collector.yaml b/examples/switchyard/otel-collector.yaml deleted file mode 100644 index 330a87077..000000000 --- a/examples/switchyard/otel-collector.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -receivers: - otlp: - protocols: - http: - endpoint: 0.0.0.0:4318 - -exporters: - file/trajectory: - path: /artifacts/trajectory.otel.json - otlp/phoenix: - endpoint: phoenix:4317 - tls: - insecure: true - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [file/trajectory, otlp/phoenix] diff --git a/examples/switchyard/plugins.toml b/examples/switchyard/plugins.toml index d6a1a69ed..a17040169 100644 --- a/examples/switchyard/plugins.toml +++ b/examples/switchyard/plugins.toml @@ -8,68 +8,28 @@ kind = "switchyard" enabled = true [components.config] -mode = "enforce" +version = 2 priority = 0 -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "smart-stage-router" -request_materialization = "recent_message_window" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 25 -max_retries = 3 -recent_message_count = 8 -enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] +max_retries = 2 +enabled_inbound_profiles = ["openai_chat"] -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +[components.config.algorithm] +kind = "random" +seed = 42 [components.config.default_targets] -openai_chat = "openai-default" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" +openai_chat = "fast" -[components.config.targets.openai-default] -model = "weak-model" +[components.config.targets.fast] +model = "provider/fast" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "http://127.0.0.1:4101" +weight = 1 -[components.config.targets.responses-default] -model = "weak-model" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "weak-model" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.strong] -model = "strong-model" +[components.config.targets.quality] +model = "provider/quality" protocol = "openai_chat" endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4102" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 3000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" +base_url = "http://127.0.0.1:4101" +weight = 1 diff --git a/examples/switchyard/real-e2e-plugins.toml b/examples/switchyard/real-e2e-plugins.toml deleted file mode 100644 index 541beb057..000000000 --- a/examples/switchyard/real-e2e-plugins.toml +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -version = 1 - -[[components]] -kind = "switchyard" -enabled = true - -[components.config] -mode = "enforce" -decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" -decision_profile_id = "remote-stage-router" -request_materialization = "summary_only" -context_mode = "atof_required" -atof_endpoint_name = "switchyard" -decision_timeout_millis = 1000 -max_retries = 3 -recent_message_count = 8 - -[components.config.decision_header_env] -authorization = "SWITCHYARD_AUTHORIZATION" - -[components.config.default_targets] -openai_chat = "weak" -openai_responses = "responses-default" -anthropic_messages = "anthropic-default" - -[components.config.targets.strong] -model = "provider/strong" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.weak] -model = "provider/weak" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.responses-default] -model = "provider/weak-responses" -protocol = "openai_responses" -endpoint = "/v1/responses" -base_url = "http://127.0.0.1:4101" - -[components.config.targets.anthropic-default] -model = "provider/weak-anthropic" -protocol = "anthropic_messages" -endpoint = "/v1/messages" -base_url = "http://127.0.0.1:4101" - -[[components]] -kind = "observability" -enabled = true - -[components.config.atof] -enabled = true - -[[components.config.atof.sinks]] -type = "file" -mode = "append" - -[[components.config.atof.sinks]] -type = "stream" -name = "switchyard" -url = "http://127.0.0.1:4000/v1/atof/events" -transport = "http_post" -field_name_policy = "preserve" -timeout_millis = 1000 - -[components.config.atof.sinks.header_env] -authorization = "SWITCHYARD_AUTHORIZATION" diff --git a/examples/switchyard/real-e2e-profiles.yaml b/examples/switchyard/real-e2e-profiles.yaml deleted file mode 100644 index 6127b46ef..000000000 --- a/examples/switchyard/real-e2e-profiles.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -targets: - strong: - model: provider/strong - format: openai - base_url: http://127.0.0.1:4101/v1 - weak: - model: provider/weak - format: openai - base_url: http://127.0.0.1:4101/v1 - -profiles: - remote-stage-router: - type: stage_router - capable: strong - efficient: weak - fallback_target_on_evict: strong - picker: efficient_first - confidence_threshold: 0.7 diff --git a/examples/switchyard/run-classifier-e2e.sh b/examples/switchyard/run-classifier-e2e.sh new file mode 100755 index 000000000..4c557bf91 --- /dev/null +++ b/examples/switchyard/run-classifier-e2e.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$relay_root/examples/switchyard/e2e-common.sh" +work_dir="$(mktemp -d)" +upstream_log="$work_dir/upstream.jsonl" + +cleanup() { + local status=$? + e2e_stop_processes + if [[ $status -eq 0 ]]; then + rm -rf "$work_dir" + else + echo "E2E logs preserved in $work_dir" >&2 + e2e_tail_logs "$work_dir" + fi +} +trap cleanup EXIT + +for dependency in cargo curl python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done + +python3 "$relay_root/examples/switchyard/fake_upstream.py" \ + --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & +e2e_add_pid "$!" + +( + cd "$relay_root" + cargo run -p nemo-relay-cli --features switchyard -- \ + --plugin-config-path "$relay_root/examples/switchyard/classifier-plugins.toml" \ + --bind 127.0.0.1:4041 +) >"$work_dir/relay.log" 2>&1 & +relay_pid="$!" +e2e_add_pid "$relay_pid" + +e2e_wait_for http://127.0.0.1:4041/healthz 240 0.25 "$relay_pid" + +curl --fail --silent http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: classifier-buffered' \ + --data-binary \ + '{"model":"client/model","stream":false,"messages":[{"role":"user","content":"classify this"}]}' \ + >"$work_dir/buffered.json" + +curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: classifier-streaming' \ + --data-binary \ + '{"model":"client/model","stream":true,"messages":[{"role":"user","content":"classify this stream"}]}' \ + >"$work_dir/stream.sse" + +python3 - "$upstream_log" "$work_dir/buffered.json" "$work_dir/stream.sse" <<'PY' +import json +import pathlib +import sys + +records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +models = [record["body"]["model"] for record in records] +expected = [ + "provider/classifier", + "provider/fast", + "provider/classifier", + "provider/fast", +] +if models != expected: + raise SystemExit(f"unexpected classifier call sequence: {models}") +for record in records: + body = record["body"] + if body["model"] == "provider/classifier": + if not isinstance(body.get("response_format"), dict): + raise SystemExit(f"classifier request omitted response_format: {body}") +buffered = json.loads(pathlib.Path(sys.argv[2]).read_text()) +if buffered["model"] != "provider/fast": + raise SystemExit(f"classifier did not select the weak target: {buffered}") +stream = pathlib.Path(sys.argv[3]).read_text() +if "fake" not in stream or "[DONE]" not in stream: + raise SystemExit(f"unexpected SSE output: {stream}") +print(f"in-process Switchyard classifier E2E passed: {models}") +PY diff --git a/examples/switchyard/run-hermes-ollama-smoke.sh b/examples/switchyard/run-hermes-ollama-smoke.sh deleted file mode 100755 index f390eb75e..000000000 --- a/examples/switchyard/run-hermes-ollama-smoke.sh +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "$relay_root/examples/switchyard/e2e-common.sh" -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" -run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" -artifact_dir="${SWITCHYARD_TRAJECTORY_DIR:-$relay_root/artifacts/hermes-switchyard-$run_id}" -token="$(e2e_random_token)" -docker_network="switchyard-e2e-$run_id" -phoenix_container="switchyard-phoenix-$run_id" -collector_container="switchyard-otel-$run_id" -phoenix_port="${SWITCHYARD_PHOENIX_PORT:-6006}" -keep_phoenix="${SWITCHYARD_KEEP_PHOENIX:-0}" -collector_running=0 -phoenix_running=0 -network_created=0 - -mkdir -p "$artifact_dir/phoenix" -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" >"$artifact_dir/switchyard-revision.txt" - -cleanup() { - local status=$? - e2e_stop_processes - if [[ $collector_running -eq 1 ]]; then - docker rm -f "$collector_container" >/dev/null 2>&1 || true - fi - if [[ $phoenix_running -eq 1 && ( $status -ne 0 || "$keep_phoenix" != "1" ) ]]; then - docker rm -f "$phoenix_container" >/dev/null 2>&1 || true - phoenix_running=0 - fi - if [[ $network_created -eq 1 && $phoenix_running -eq 0 ]]; then - docker network rm "$docker_network" >/dev/null 2>&1 || true - fi - if [[ $status -ne 0 ]]; then - echo "Hermes/StageRouter smoke failed; artifacts preserved in $artifact_dir" >&2 - e2e_tail_logs "$artifact_dir" - fi -} -trap cleanup EXIT - -for dependency in cargo curl docker hermes jq python3 tar; do - command -v "$dependency" >/dev/null || { - echo "missing required command: $dependency" >&2 - exit 1 - } -done - -docker info >/dev/null -for model in llama3.2:latest qwen3.6:35b; do - curl --fail --silent http://127.0.0.1:11434/api/tags \ - | jq -e --arg model "$model" '.models[] | select(.name == $model)' >/dev/null || { - echo "required Ollama model is not installed: $model" >&2 - exit 1 - } -done - -docker network create "$docker_network" >/dev/null -network_created=1 -docker run --detach --rm \ - --name "$phoenix_container" \ - --network "$docker_network" \ - --network-alias phoenix \ - --publish "127.0.0.1:$phoenix_port:6006" \ - --env PHOENIX_WORKING_DIR=/mnt/data \ - --volume "$artifact_dir/phoenix:/mnt/data" \ - arizephoenix/phoenix:13.22 >"$artifact_dir/phoenix.container-id" -phoenix_running=1 -e2e_wait_for "http://127.0.0.1:$phoenix_port/" 240 0.5 - -docker run --detach --rm \ - --name "$collector_container" \ - --network "$docker_network" \ - --publish 127.0.0.1:4318:4318 \ - --volume "$relay_root/examples/switchyard/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \ - --volume "$artifact_dir:/artifacts" \ - otel/opentelemetry-collector-contrib:0.135.0 \ - --config=/etc/otelcol-contrib/config.yaml >"$artifact_dir/collector.container-id" -collector_running=1 - -( - cd "$switchyard_root" - OLLAMA_CLASSIFIER_API_KEY="$token" \ - SWITCHYARD_ATOF_BEARER_TOKEN="$token" \ - cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/hermes-ollama-profiles.yaml" --port 4000 -) >"$artifact_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" -e2e_wait_for http://127.0.0.1:4000/health 240 0.5 - -run_query() { - local sequence="$1" - local label="$2" - local query="$3" - local resume_id="${4:-}" - local -a resume_args=() - local before_lines=0 - local after_lines - local atif_path - if [[ -n "$resume_id" ]]; then - resume_args=(--resume "$resume_id") - fi - if [[ -f "$artifact_dir/trajectory.atof.jsonl" ]]; then - before_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - fi - ( - cd "$artifact_dir" - HERMES_HOME="$artifact_dir/hermes" \ - OPENAI_API_KEY=ollama \ - SWITCHYARD_AUTHORIZATION="Bearer $token" \ - cargo run --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli \ - --features switchyard -- \ - run --agent hermes \ - --plugin-config-path "$relay_root/examples/switchyard/hermes-ollama-plugins.toml" \ - -- chat --provider custom --model llama3.2:latest \ - --query "$query" ${resume_args[@]+"${resume_args[@]}"} \ - --toolsets terminal --quiet --max-turns 2 --ignore-rules - ) >"$artifact_dir/query-$sequence-$label.log" 2>&1 - after_lines="$(wc -l < "$artifact_dir/trajectory.atof.jsonl" | tr -d ' ')" - printf '%s\t%s\t%s\t%s\n' "$sequence" "$label" "$before_lines" "$after_lines" \ - >> "$artifact_dir/query-event-ranges.tsv" - # Each query is a separate Relay process. Replay its persisted ATOF segment - # as a completion barrier before the next process starts; ingestion is - # idempotent, so events already delivered by the best-effort live exporter - # are reported as duplicates rather than applied twice. - sed -n "$((before_lines + 1)),${after_lines}p" "$artifact_dir/trajectory.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$artifact_dir/trajectory-$sequence-$label.atof.jsonl" \ - > "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" - jq -e '(.batch.ingested_events + .batch.duplicate_events) >= 1' \ - "$artifact_dir/trajectory-$sequence-$label.atof-ingest.json" >/dev/null - atif_path="$(find "$artifact_dir" -maxdepth 1 -name 'trajectory-*.atif.json' -print \ - | grep -E '/trajectory-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.atif\.json$' \ - | head -1 || true)" - if [[ -z "$atif_path" ]]; then - echo "ATIF exporter did not produce a trajectory for query $sequence" >&2 - exit 1 - fi - mv "$atif_path" "$artifact_dir/trajectory-$sequence-$label.atif.json" -} - -emit_stage_router_signal() { - local sequence="$1" - local label="$2" - local output="$3" - local path="$artifact_dir/trajectory-signal-$sequence-$label.atof.jsonl" - python3 - "$path" "$session_id" "$label" "$output" <<'PY' -import datetime -import json -import pathlib -import sys -import uuid - -path = pathlib.Path(sys.argv[1]) -session_id, label, output = sys.argv[2:] -event_uuid = str(uuid.uuid4()) -base = { - "atof_version": "0.1", - "kind": "scope", - "uuid": event_uuid, - "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "name": "trajectory_fixture", - "category": "tool", - "category_profile": {"tool_call_id": f"fixture-{label}"}, - "metadata": { - "session_id": session_id, - "trajectory_fixture": True, - "trajectory_fixture_label": label, - }, -} -events = [ - {**base, "scope_category": "start", "data": {"label": label}}, - {**base, "scope_category": "end", "data": {"output": output}}, -] -path.write_text("".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events)) -PY - cat "$path" >> "$artifact_dir/trajectory.atof.jsonl" - curl --fail --silent http://127.0.0.1:4000/v1/atof/events \ - -H "authorization: Bearer $token" \ - -H 'content-type: application/x-ndjson' \ - --data-binary "@$path" > "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" - jq -e '.batch.ingested_events == 2' \ - "$artifact_dir/trajectory-signal-$sequence-$label.atof-ingest.json" >/dev/null -} - -simple_query='Return exactly the integer result of 17 + 25, with no explanation.' -complex_query='Do not call tools. Act as a principal concurrency engineer. Analyze a bounded lock-free MPMC queue that uses compare-and-swap on head and tail but no generation counters. Give a concrete ABA failure interleaving, identify the required C++ memory order on each publication and consumption edge, and propose the smallest defensible correction. Be precise about the linearization points.' -followup_query='Ignore the earlier technical topic. Reply with exactly SIMPLE_DONE and nothing else.' - -run_query 01 simple "$simple_query" -session_id="$(jq -r 'select(.name == "switchyard.routing.requested") | .data.session_id' "$artifact_dir/trajectory.atof.jsonl" | head -1)" -if [[ -z "$session_id" || "$session_id" == "null" ]]; then - echo "could not recover the Hermes session ID from the first routing mark" >&2 - exit 1 -fi -emit_stage_router_signal 02 complex \ - 'CUDA out of memory while analyzing the concurrent queue; critical failure requires careful recovery and a capable model.' -run_query 02 complex "$complex_query" "$session_id" -emit_stage_router_signal 03 simple-followup \ - 'All tests passed. The next request is a direct low-risk formatting response: return exactly SIMPLE_DONE.' -run_query 03 simple-followup "$followup_query" "$session_id" - -# Give the asynchronous OTLP exporter a short flush window, then stop the -# collector cleanly so its file exporter closes the shareable OTLP JSON file. -sleep 2 -docker stop --time 10 "$collector_container" >/dev/null -collector_running=0 - -python3 - "$artifact_dir" "$session_id" "$phoenix_port" "$simple_query" "$complex_query" "$followup_query" <<'PY' -import json -import pathlib -import sys - -root = pathlib.Path(sys.argv[1]) -session_id = sys.argv[2] -phoenix_port = sys.argv[3] -queries = sys.argv[4:] -atof_path = root / "trajectory.atof.jsonl" -events = [json.loads(line) for line in atof_path.read_text().splitlines() if line.strip()] -marks = [event for event in events if event.get("name", "").startswith("switchyard.routing.")] -decisions = [event for event in marks if event.get("name") == "switchyard.routing.decision"] - -expected_models = ["llama3.2:latest", "qwen3.6:35b", "llama3.2:latest"] -event_ranges = [] -for line in (root / "query-event-ranges.tsv").read_text().splitlines(): - sequence, label, start, end = line.split("\t") - event_ranges.append((sequence, label, int(start), int(end))) -representative_decisions = [] -for sequence, label, start, end in event_ranges: - segment = events[start:end] - decision = next( - (event for event in segment if event.get("name") == "switchyard.routing.decision"), - None, - ) - if decision is None: - raise SystemExit(f"query {sequence} ({label}) produced no successful routing decision") - representative_decisions.append(decision) -actual_models = [event.get("data", {}).get("selected_model") for event in representative_decisions] -if actual_models != expected_models: - raise SystemExit(f"unexpected StageRouter route sequence: {actual_models}; expected {expected_models}") - -required_mark_names = {"switchyard.routing.requested", "switchyard.routing.decision"} -if not required_mark_names.issubset({event.get("name") for event in marks}): - raise SystemExit("routing requested/decision marks were not both emitted") -for event in marks: - name = event.get("name") - if event.get("category") != "custom": - raise SystemExit(f"{name} did not use category=custom") - if event.get("category_profile", {}).get("subtype") != name: - raise SystemExit(f"{name} category_profile.subtype was not canonical") - schema = event.get("data_schema", {}) - if schema != {"name": "switchyard.routing_mark", "version": "1"}: - raise SystemExit(f"{name} had unexpected data_schema: {schema}") - metadata = event.get("metadata", {}) - if metadata.get("session_id") != session_id: - raise SystemExit(f"{name} did not mirror session identity") -for event in decisions: - data = event["data"] - for key in ("decision_id", "router", "routing_attempt", "backend_id", "selected_tier", "selected_model", "latency_ms", "rollout_mode"): - if key not in data or data[key] is None: - raise SystemExit(f"decision mark missing {key}: {data}") - -atif_paths = sorted(root.glob("trajectory-*.atif.json")) -if len(atif_paths) != 3: - raise SystemExit(f"expected three ATIF trajectories, found {len(atif_paths)}") -for path in atif_paths: - payload = json.loads(path.read_text()) - if not payload.get("steps"): - raise SystemExit(f"ATIF trajectory has no steps: {path.name}") - -otel_path = root / "trajectory.otel.json" -if not otel_path.exists() or not otel_path.read_text().strip(): - raise SystemExit("OTLP file exporter did not produce trajectory.otel.json") -otel_batches = [json.loads(line) for line in otel_path.read_text().splitlines() if line.strip()] - -summary = { - "session_id": session_id, - "queries": [{ - "sequence": index + 1, - "input": query, - "selected_model": actual_models[index], - "selected_tier": representative_decisions[index]["data"].get("selected_tier"), - "reason_code": representative_decisions[index]["data"].get("reason_code"), - "reason_summary": representative_decisions[index]["data"].get("reason_summary"), - } for index, query in enumerate(queries)], - "expected_route_sequence": expected_models, - "actual_route_sequence": actual_models, - "routing_basis": [ - "cold StageRouter efficient default", - "canonical ATOF critical-error tool result (capable override)", - "canonical ATOF clean-tests tool result (efficient classifier decision)", - ], - "atof": {"file": atof_path.name, "event_count": len(events), "routing_mark_count": len(marks)}, - "atif": {"files": [path.name for path in atif_paths], "trajectory_count": len(atif_paths)}, - "otel": {"file": otel_path.name, "export_batch_count": len(otel_batches)}, - "phoenix_url": f"http://127.0.0.1:{phoenix_port}", -} -(root / "trajectory-summary.json").write_text(json.dumps(summary, indent=2) + "\n") -readme = f"""# Hermes / Ollama / Switchyard StageRouter trajectory - -This bundle captures one fixed three-query Hermes session routed through NeMo -Relay and the Switchyard Decision API. The verified representative route is: - -1. `llama3.2:latest` (efficient) — cold StageRouter default -2. `qwen3.6:35b` (capable) — critical-signal StageRouter override -3. `llama3.2:latest` (efficient) — clean-state classifier decision - -Session ID: `{session_id}` - -## Important fixture note - -The `CUDA out of memory` text in `trajectory-signal-02-complex.atof.jsonl` is an -intentional, synthetic ATOF tool-result fixture. The machine did not run out of -memory. Fixture events carry `metadata.trajectory_fixture = true` and a fixture -label so they cannot be confused with organic Hermes events. - -The fixtures are necessary for this demonstration because the current -Switchyard StageRouter Decision API classifies from its accumulated ATOF snapshot, -not directly from `current_request.body`. The critical fixture exercises the -real capable override. The clean-tests fixture removes the prior critical signal -from the one-result window and exercises the efficient classifier path. - -## File map - -| Files | Contents | Test coverage | -| --- | --- | --- | -| `trajectory-summary.json` | Machine-readable queries, selected models, reasons, counts, and Phoenix URL | Confirms expected and actual representative routes match | -| `trajectory.atof.jsonl` | Complete Relay ATOF stream for all three queries and the labeled fixtures | Identity propagation, lifecycle events, routing marks, and accumulator input | -| `trajectory-01-simple.atof.jsonl` | ATOF emitted by the first Hermes invocation | Cold-start efficient default | -| `trajectory-02-complex.atof.jsonl` | ATOF emitted by the complex Hermes invocation | Dispatch through the selected capable backend | -| `trajectory-03-simple-followup.atof.jsonl` | ATOF emitted by the final Hermes invocation | Return to the efficient backend | -| `trajectory-signal-*.atof.jsonl` | Canonical, labeled tool start/end fixtures | Capable critical-error override and efficient clean-state classification | -| `*.atof-ingest.json` | Switchyard ingestion reports for query segments and fixtures | Successful or idempotent ATOF accumulation | -| `trajectory-01-simple.atif.json` | ATIF representation of query 1 | Efficient-model trajectory structure | -| `trajectory-02-complex.atif.json` | ATIF representation of query 2 | Capable-model trajectory structure | -| `trajectory-03-simple-followup.atif.json` | ATIF representation of query 3 | Efficient follow-up trajectory structure | -| `trajectory.otel.json` | OTLP JSON batches written by the OpenTelemetry Collector | Relay spans exported to the collector and forwarded to Phoenix | -| `query-*.log` | Hermes/Relay stdout and stderr for each invocation | Human-readable harness responses and execution diagnostics | -| `query-event-ranges.tsv` | Query label and ATOF line-count boundaries | Separates representative user-query decisions from extra Hermes calls | - -## Routing-mark assertions - -The smoke validates every `switchyard.routing.*` mark in the cumulative ATOF -stream. Each mark must have: - -- `category: "custom"` -- `category_profile.subtype` equal to the mark name -- `data_schema.name: "switchyard.routing_mark"` -- `data_schema.version: "1"` -- the expected session identity in metadata - -Decision marks must also include a decision ID, router, attempt, backend, tier, -model, latency, and rollout mode. This run produced {len(marks)} routing marks -across {len(events)} total ATOF events. - -## Phoenix and OTLP - -During the smoke, Relay sends OTLP/HTTP to a local OpenTelemetry Collector. The -collector writes `trajectory.otel.json` and forwards the same spans to Phoenix. -The run produced {len(otel_batches)} OTLP export batches. When the smoke is run -with `SWITCHYARD_KEEP_PHOENIX=1`, open the `phoenix_url` from -`trajectory-summary.json` before stopping the printed Phoenix container. - -## Reproduce - -Install both `llama3.2:latest` and `qwen3.6:35b` in Ollama, ensure the cumulative -Switchyard checkout is available, then run: - -```bash -SWITCHYARD_KEEP_PHOENIX=1 examples/switchyard/run-hermes-ollama-smoke.sh -``` - -The script validates the route, mark shape, ATIF contents, and OTLP output before -creating this bundle. -""" -(root / "TRAJECTORY_README.md").write_text(readme) -print(json.dumps(summary, indent=2)) -PY - -( - cd "$artifact_dir" - tar -czf trajectory-bundle.tar.gz \ - TRAJECTORY_README.md \ - trajectory-summary.json \ - trajectory.atof.jsonl \ - trajectory.otel.json \ - trajectory-*.atif.json \ - trajectory-*.atof.jsonl \ - trajectory-*.atof-ingest.json \ - query-*.log \ - query-event-ranges.tsv -) - -echo "Hermes/Ollama StageRouter trajectory passed: llama3.2 -> qwen3.6:35b -> llama3.2" -echo "Artifacts: $artifact_dir" -echo "Bundle: $artifact_dir/trajectory-bundle.tar.gz" -if [[ "$keep_phoenix" == "1" ]]; then - echo "Phoenix: http://127.0.0.1:$phoenix_port (container $phoenix_container left running)" -else - echo "Set SWITCHYARD_KEEP_PHOENIX=1 to leave Phoenix running after the smoke." -fi diff --git a/examples/switchyard/run-real-e2e.sh b/examples/switchyard/run-real-e2e.sh index 1f6810871..1d20fd99e 100755 --- a/examples/switchyard/run-real-e2e.sh +++ b/examples/switchyard/run-real-e2e.sh @@ -6,19 +6,8 @@ set -euo pipefail relay_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "$relay_root/examples/switchyard/e2e-common.sh" -relay_toolchain="" -if command -v rustup >/dev/null 2>&1; then - relay_toolchain="$(cd "$relay_root" && rustup show active-toolchain)" - relay_toolchain="${relay_toolchain%% *}" -fi -switchyard_root="${SWITCHYARD_ROOT:-$(cd "$relay_root/.." && pwd)/Switchyard-topic-nemo-relay-integration}" -switchyard_expected_commit="${SWITCHYARD_EXPECTED_COMMIT:-8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8}" work_dir="$(mktemp -d)" upstream_log="$work_dir/upstream.jsonl" -token="$(e2e_random_token)" - -[[ -d "$switchyard_root" ]] || { echo "Switchyard worktree not found: $switchyard_root" >&2; exit 1; } -e2e_verify_switchyard_checkout "$switchyard_root" "$switchyard_expected_commit" cleanup() { local status=$? @@ -32,58 +21,45 @@ cleanup() { } trap cleanup EXIT +for dependency in cargo curl python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done + python3 "$relay_root/examples/switchyard/fake_upstream.py" \ --port 4101 --log "$upstream_log" >"$work_dir/upstream.log" 2>&1 & e2e_add_pid "$!" ( - cd "$switchyard_root" - SWITCHYARD_ATOF_BEARER_TOKEN="$token" cargo run -p switchyard-server -- \ - --config "$relay_root/examples/switchyard/real-e2e-profiles.yaml" --port 4000 -) >"$work_dir/switchyard.log" 2>&1 & -e2e_add_pid "$!" - -e2e_wait_for http://127.0.0.1:4000/health - -( - if [[ -n "$relay_toolchain" ]]; then - export RUSTUP_TOOLCHAIN="$relay_toolchain" - fi - cd "$work_dir" - SWITCHYARD_AUTHORIZATION="Bearer $token" cargo run \ - --manifest-path "$relay_root/Cargo.toml" -p nemo-relay-cli --features switchyard -- \ - --plugin-config-path "$relay_root/examples/switchyard/real-e2e-plugins.toml" \ + cd "$relay_root" + cargo run -p nemo-relay-cli --features switchyard -- \ + --plugin-config-path "$relay_root/examples/switchyard/plugins.toml" \ --bind 127.0.0.1:4041 ) >"$work_dir/relay.log" 2>&1 & relay_pid="$!" e2e_add_pid "$relay_pid" -e2e_wait_for http://127.0.0.1:4041/healthz 120 0.25 "$relay_pid" +e2e_wait_for http://127.0.0.1:4041/healthz 240 0.25 "$relay_pid" -request() { - local request_id="$1" - local stream="$2" - curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ +for request in $(seq 1 8); do + curl --fail --silent http://127.0.0.1:4041/v1/chat/completions \ -H 'content-type: application/json' \ - -H 'x-nemo-relay-session-id: e2e-session' \ - -H "x-nemo-relay-request-id: $request_id" \ - --data-binary "{\"model\":\"client-model\",\"stream\":$stream,\"messages\":[{\"role\":\"user\",\"content\":\"hello\"}]}" -} - -request cold-request false >"$work_dir/cold.json" - -for payload in \ - '{"hook_event_name":"on_session_start","session_id":"e2e-session"}' \ - '{"hook_event_name":"pre_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' \ - '{"hook_event_name":"post_tool_call","session_id":"e2e-session","tool_name":"Bash","tool_input":{"command":"test"},"tool_response":{"output":"CUDA out of memory"},"extra":{"task_id":"task-1","tool_call_id":"call-1"}}' -do - curl --fail --silent http://127.0.0.1:4041/hooks/hermes \ - -H 'content-type: application/json' --data-binary "$payload" >/dev/null + -H 'x-nemo-relay-session-id: libsy-example' \ + -H "x-nemo-relay-request-id: buffered-$request" \ + --data-binary \ + '{"model":"client/model","stream":false,"messages":[{"role":"user","content":"hello"}]}' \ + >"$work_dir/buffered-$request.json" done -sleep 1 -request warm-request false >"$work_dir/warm.json" -request stream-request true >"$work_dir/stream.sse" +curl --fail --silent --no-buffer http://127.0.0.1:4041/v1/chat/completions \ + -H 'content-type: application/json' \ + -H 'x-nemo-relay-session-id: libsy-example' \ + -H 'x-nemo-relay-request-id: stream-1' \ + --data-binary \ + '{"model":"client/model","stream":true,"messages":[{"role":"user","content":"stream hello"}]}' \ + >"$work_dir/stream.sse" python3 - "$upstream_log" "$work_dir/stream.sse" <<'PY' import json @@ -92,10 +68,12 @@ import sys records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] models = [record["body"]["model"] for record in records] -if models != ["provider/weak", "provider/strong", "provider/strong"]: - raise SystemExit(f"unexpected cold/warm/stream route sequence: {models}") +if set(models) != {"provider/fast", "provider/quality"}: + raise SystemExit(f"seeded libsy router did not exercise both targets: {models}") stream = pathlib.Path(sys.argv[2]).read_text() if "fake" not in stream or "[DONE]" not in stream: raise SystemExit(f"unexpected SSE output: {stream}") -print(f"real Switchyard E2E passed: {models}") +if '"system_fingerprint":"fp_switchyard_example"' not in stream: + raise SystemExit(f"provider stream extension was not preserved: {stream}") +print(f"in-process Switchyard libsy E2E passed: {models}") PY