diff --git a/docs/release_notes.md b/docs/release_notes.md index 449e16543..e16ac20e9 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -16,7 +16,8 @@ ### 📈 Improvements --[Orchestration] Added new API `OrchestrationTemplateReference#withScope` to support prompt templates with resource-group scope. +- [Orchestration] Added new API `OrchestrationTemplateReference#withScope` to support prompt templates with resource-group scope. +- [Orchestration] Chat completion calls now can have multiple module configs to support [fallback modules](https://sap.github.io/ai-sdk/docs/java/orchestration/chat-completion). ### 🐛 Fixed Issues diff --git a/orchestration/pom.xml b/orchestration/pom.xml index b75bbd7b2..0a3292b12 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -40,7 +40,7 @@ 95% 93% 74% - 94% + 95% 100% diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java index b5725c6ee..1a0170c6a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java @@ -17,6 +17,9 @@ import com.sap.ai.sdk.orchestration.model.TranslationModuleConfig; import io.vavr.control.Option; import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.IntStream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -30,24 +33,41 @@ final class ConfigToRequestTransformer { @Nonnull static CompletionRequestConfiguration toCompletionPostRequest( - @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) { - val template = toTemplateModuleConfig(prompt, config.getTemplateConfig()); + @Nonnull final OrchestrationPrompt prompt, + @Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { + + final List configList = new ArrayList<>(); + configList.add(config); + if (fallbackConfigs.length > 0) { + configList.addAll(Arrays.asList(fallbackConfigs)); + } + val templates = + configList.stream() + .map(conf -> toTemplateModuleConfig(prompt, conf.getTemplateConfig())) + .toList(); // note that the config is immutable and implicitly copied here // copying is required here, to not alter the original config object, which might be reused for // subsequent requests - val configCopy = config.withTemplateConfig(template); + val configsCopy = + IntStream.range(0, configList.size()) + .mapToObj(i -> configList.get(i).withTemplateConfig(templates.get(i))) + .toList(); val messageHistory = prompt.getMessagesHistory().stream().map(Message::createChatMessage).toList(); - val moduleConfigs = toModuleConfigs(configCopy); + final OrchestrationConfigModules moduleConfigs = + configsCopy.size() == 1 + ? toModuleConfigs(configsCopy.get(0)) + : toListOfModuleConfigs(configsCopy); - val reqConfig = + val requestConfig = OrchestrationConfig.create().modules(moduleConfigs).stream(config.getGlobalStreamOptions()); return CompletionRequestConfiguration.create() - .config(reqConfig) + .config(requestConfig) .placeholderValues(prompt.getTemplateParameters()) .messagesHistory(messageHistory); } @@ -93,6 +113,18 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( @Nonnull static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConfig config) { + return OrchestrationConfigModules.createInnerModuleConfigs(setupModuleConfigs(config)); + } + + @Nonnull + static OrchestrationConfigModules.ListOfModuleConfigss toListOfModuleConfigs( + @Nonnull final List configs) { + return OrchestrationConfigModules.createListOfModuleConfigss( + configs.stream().map(ConfigToRequestTransformer::setupModuleConfigs).toList()); + } + + @Nonnull + private static ModuleConfigs setupModuleConfigs(@Nonnull final OrchestrationModuleConfig config) { val llmConfig = Option.of(config.getLlmConfig()) .getOrElseThrow(() -> new IllegalStateException("LLM config is required.")); @@ -117,8 +149,7 @@ static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConf inputTranslation.forEach(moduleConfig.getTranslation()::input); outputTranslation.forEach(moduleConfig.getTranslation()::output); } - - return OrchestrationConfigModules.createInnerModuleConfigs(moduleConfig); + return moduleConfig; } @Nonnull diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 66fe58bf1..80617b2d7 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -14,7 +14,6 @@ import com.sap.ai.sdk.orchestration.model.EmbeddingsPostRequest; import com.sap.ai.sdk.orchestration.model.EmbeddingsPostResponse; import com.sap.ai.sdk.orchestration.model.GlobalStreamOptions; -import com.sap.ai.sdk.orchestration.model.ModuleConfigs; import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; @@ -74,28 +73,33 @@ public OrchestrationClient(@Nonnull final HttpDestination destination) { * * @param prompt The {@link OrchestrationPrompt} to generate a completion for. * @param config The {@link OrchestrationConfig } configuration to use for the completion. + * @param fallbackConfigs Fallback configurations to use. * @return The low-level request data object to send to orchestration. */ @Nonnull public static CompletionRequestConfiguration toCompletionPostRequest( - @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) { - return ConfigToRequestTransformer.toCompletionPostRequest(prompt, config); + @Nonnull final OrchestrationPrompt prompt, + @Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { + return ConfigToRequestTransformer.toCompletionPostRequest(prompt, config, fallbackConfigs); } /** * Generate a completion for the given prompt. * * @param prompt The {@link OrchestrationPrompt} to send to orchestration. - * @param config The {@link ModuleConfigs} configuration to use for the completion. + * @param config the configuration to use + * @param fallbackConfigs fallback configurations * @return the completion output * @throws OrchestrationClientException if the request fails. */ @Nonnull public OrchestrationChatResponse chatCompletion( - @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) + @Nonnull final OrchestrationPrompt prompt, + @Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) throws OrchestrationClientException { - - val request = toCompletionPostRequest(prompt, config); + val request = toCompletionPostRequest(prompt, config, fallbackConfigs); val response = executeRequest(request); return new OrchestrationChatResponse(response); } @@ -105,6 +109,7 @@ public OrchestrationChatResponse chatCompletion( * * @param prompt a text message. * @param config the configuration to use + * @param fallbackConfigs fallback configurations * @return a stream of message deltas * @throws OrchestrationClientException if the request fails or if the finish reason is * content_filter @@ -112,10 +117,12 @@ public OrchestrationChatResponse chatCompletion( */ @Nonnull public Stream streamChatCompletion( - @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) + @Nonnull final OrchestrationPrompt prompt, + @Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) throws OrchestrationClientException { - val request = toCompletionPostRequest(prompt, config); + val request = toCompletionPostRequest(prompt, config, fallbackConfigs); return streamChatCompletionDeltas(request) .peek(OrchestrationClient::throwOnContentFilter) .map(OrchestrationChatCompletionDelta::getDeltaContent); diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java index 2d6ba8811..a3dec5a90 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java @@ -1504,4 +1504,45 @@ void testGetAllMessages() { assertThat(messageListTools.get(1)).isInstanceOf(AssistantMessage.class); assertThat(messageListTools.get(2)).isInstanceOf(ToolMessage.class); } + + @Test + void testFallbackModules() throws IOException { + stubFor( + post(urlPathEqualTo("/v2/completion")) + .willReturn( + aResponse() + .withBodyFile("fallbackResponse.json") + .withHeader("Content-Type", "application/json"))); + + final var prompt = new OrchestrationPrompt("HelloWorld! Why is this phrase so famous?"); + final var llamaFilter = new LlamaGuardFilter().config(LlamaGuard38b.create().selfHarm(true)); + val groundingConfig = + Grounding.create() + .filters( + DocumentGroundingFilter.create() + .dataRepositoryType(DataRepositoryType.HELP_SAP_COM)); + + final var workingConfig = + new OrchestrationModuleConfig() + .withLlmConfig(GPT_4O_MINI.withParam(TEMPERATURE, 0.0)) + .withInputFiltering(llamaFilter) + .withGrounding(groundingConfig); + final var brokenConfig = + workingConfig.withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + + final var response = client.chatCompletion(prompt, brokenConfig, workingConfig); + + var intermediateFailure = response.getOriginalResponse().getIntermediateFailures().get(0); + assertThat(intermediateFailure.getCode()).isEqualTo(400); + assertThat(intermediateFailure.getLocation()).isEqualTo("Request Body"); + assertThat(intermediateFailure.getRequestId()) + .isEqualTo("a562703b-7fe5-97ba-b417-e8140a25fb7c"); + assertThat(intermediateFailure.getMessage()) + .isEqualTo("400 - Request Body: Model broken_name not supported."); + assertThat(intermediateFailure.getHeaders().get("Content-Type")).isEqualTo("application/json"); + + final String request = fileLoaderStr.apply("fallbackRequest.json"); + verify( + postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request))); + } } diff --git a/orchestration/src/test/resources/__files/fallbackResponse.json b/orchestration/src/test/resources/__files/fallbackResponse.json new file mode 100644 index 000000000..64852a79f --- /dev/null +++ b/orchestration/src/test/resources/__files/fallbackResponse.json @@ -0,0 +1,81 @@ +{ + "request_id": "a562703b-7fe5-97ba-b417-e8140a25fb7c", + "intermediate_results": { + "templating": [ + { + "content": "HelloWorld! Why is this phrase so famous?", + "role": "user" + } + ], + "llm": { + "id": "", + "object": "chat.completion", + "created": 1769700208, + "model": "gemini-2.5-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "\"Hello, World!\" is arguably the most famous phrase in the world of programming, and its fame stems from a combination of historical significance, pedagogical utility, and cultural impact.\n\nHere's why it's so famous:\n\n1. **Historical Origin and Standardization:**\n * While simple output programs existed before, the phrase \"Hello, World!\" was popularized and standardized by **Brian Kernighan and Dennis Ritchie** in their seminal 1978 book, **\"The C Programming Language.\"**\n * In the book, the very first example program printed \"hello, world\" (lowercase, no exclamation mark initially). This book became the definitive guide for C, and C became one of the most influential programming languages ever. As C spread, so did its iconic first program.\n * Before K&R, a similar example (\"hello\") appeared in a 1974 Bell Labs internal memo by Kernighan, demonstrating the BCPL language.\n\n2. **Pedagogical Value (Teaching Tool):**\n * **First Step for Beginners:** It's the absolute simplest program that actually *does* something visible. For someone learning to code, successfully running \"Hello, World!\" is an immediate, tangible victory.\n * **Low Barrier to Entry:** It requires minimal syntax and no complex concepts (like variables, loops, functions, or data structures). It focuses solely on the core idea of output.\n * **Immediate Gratification:** Seeing \"Hello, World!\" appear on the screen provides instant feedback and motivation, confirming that the development environment is set up correctly and the basic compilation/execution process works.\n * **Introduction to Core Concepts:** It subtly introduces fundamental ideas like:\n * The concept of a program.\n * The idea of output (printing to the console).\n * The basic structure of a program in a given language.\n\n3. **Practical Utility (Testing and Verification):**\n * **Sanity Check:** When setting up a new programming language, a new development environment (IDE), a new compiler, or a new operating system, \"Hello, World!\" is the go-to program to verify that everything is working correctly. If \"Hello, World!\" doesn't run, you know the problem is fundamental to your setup, not your complex code.\n * **Minimal Viable Product (MVP) for Setup:** It's the quickest way to confirm that the toolchain (editor, compiler/interpreter, linker, runtime) is functional.\n\n4. **Cultural Significance and Rite of Passage:**\n * **Universal Rite of Passage:** Every programmer, regardless of the language they learn, almost certainly starts with \"Hello, World!\" It's a shared experience that connects developers across generations and technologies.\n * **Symbol of Beginning:** It represents the very first step into the vast and complex world of programming. It's a literal \"hello\" to the machine and the coding journey.\n * **Ubiquity:** Its presence in countless tutorials, textbooks, and online guides for virtually every programming language has cemented its status as an iconic and universally recognized phrase.\n\nIn essence, \"Hello, World!\" is famous because it's simple, effective, historically significant, and serves as a universal welcome mat for anyone stepping into the world of computer programming.", + "tool_calls": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "completion_tokens": 1728, + "prompt_tokens": 9, + "total_tokens": 1737, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 1024 + } + }, + "citations": [] + }, + "output_unmasking": [] + }, + "final_result": { + "id": "", + "object": "chat.completion", + "created": 1769700208, + "model": "gemini-2.5-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "\"Hello, World!\" is arguably the most famous phrase in the world of programming, and its fame stems from a combination of historical significance, pedagogical utility, and cultural impact.\n\nHere's why it's so famous:\n\n1. **Historical Origin and Standardization:**\n * While simple output programs existed before, the phrase \"Hello, World!\" was popularized and standardized by **Brian Kernighan and Dennis Ritchie** in their seminal 1978 book, **\"The C Programming Language.\"**\n * In the book, the very first example program printed \"hello, world\" (lowercase, no exclamation mark initially). This book became the definitive guide for C, and C became one of the most influential programming languages ever. As C spread, so did its iconic first program.\n * Before K&R, a similar example (\"hello\") appeared in a 1974 Bell Labs internal memo by Kernighan, demonstrating the BCPL language.\n\n2. **Pedagogical Value (Teaching Tool):**\n * **First Step for Beginners:** It's the absolute simplest program that actually *does* something visible. For someone learning to code, successfully running \"Hello, World!\" is an immediate, tangible victory.\n * **Low Barrier to Entry:** It requires minimal syntax and no complex concepts (like variables, loops, functions, or data structures). It focuses solely on the core idea of output.\n * **Immediate Gratification:** Seeing \"Hello, World!\" appear on the screen provides instant feedback and motivation, confirming that the development environment is set up correctly and the basic compilation/execution process works.\n * **Introduction to Core Concepts:** It subtly introduces fundamental ideas like:\n * The concept of a program.\n * The idea of output (printing to the console).\n * The basic structure of a program in a given language.\n\n3. **Practical Utility (Testing and Verification):**\n * **Sanity Check:** When setting up a new programming language, a new development environment (IDE), a new compiler, or a new operating system, \"Hello, World!\" is the go-to program to verify that everything is working correctly. If \"Hello, World!\" doesn't run, you know the problem is fundamental to your setup, not your complex code.\n * **Minimal Viable Product (MVP) for Setup:** It's the quickest way to confirm that the toolchain (editor, compiler/interpreter, linker, runtime) is functional.\n\n4. **Cultural Significance and Rite of Passage:**\n * **Universal Rite of Passage:** Every programmer, regardless of the language they learn, almost certainly starts with \"Hello, World!\" It's a shared experience that connects developers across generations and technologies.\n * **Symbol of Beginning:** It represents the very first step into the vast and complex world of programming. It's a literal \"hello\" to the machine and the coding journey.\n * **Ubiquity:** Its presence in countless tutorials, textbooks, and online guides for virtually every programming language has cemented its status as an iconic and universally recognized phrase.\n\nIn essence, \"Hello, World!\" is famous because it's simple, effective, historically significant, and serves as a universal welcome mat for anyone stepping into the world of computer programming.", + "tool_calls": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "completion_tokens": 1728, + "prompt_tokens": 9, + "total_tokens": 1737, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 1024 + } + }, + "citations": [] + }, + "intermediate_failures": [ + { + "request_id": "a562703b-7fe5-97ba-b417-e8140a25fb7c", + "code": 400, + "message": "400 - Request Body: Model broken_name not supported.", + "location": "Request Body", + "headers": { + "Content-Type": "application/json" + } + } + ] +} \ No newline at end of file diff --git a/orchestration/src/test/resources/fallbackRequest.json b/orchestration/src/test/resources/fallbackRequest.json new file mode 100644 index 000000000..3c378e4a8 --- /dev/null +++ b/orchestration/src/test/resources/fallbackRequest.json @@ -0,0 +1,97 @@ +{ + "config" : { + "modules" : [ { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "HelloWorld! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "broken_name", + "version" : "latest", + "params" : { }, + "timeout" : 600, + "max_retries" : 2 + } + }, + "filtering" : { + "input" : { + "filters" : [ { + "type" : "llama_guard_3_8b", + "config" : { + "self_harm" : true + } + } ] + } + }, + "grounding" : { + "type" : "document_grounding_service", + "config" : { + "filters" : [ { + "data_repositories" : [ "*" ], + "data_repository_type" : "help.sap.com", + "data_repository_metadata" : [ ], + "document_metadata" : [ ], + "chunk_metadata" : [ ] + } ], + "placeholders" : { + "input" : [ "userMessage" ], + "output" : "groundingContext" + } + } + } + }, { + "prompt_templating" : { + "prompt" : { + "template" : [ { + "content" : "HelloWorld! Why is this phrase so famous?", + "role" : "user" + } ], + "defaults" : { }, + "tools" : [ ] + }, + "model" : { + "name" : "gpt-4o-mini", + "version" : "latest", + "params" : { + "temperature" : 0.0 + }, + "timeout" : 600, + "max_retries" : 2 + } + }, + "filtering" : { + "input" : { + "filters" : [ { + "type" : "llama_guard_3_8b", + "config" : { + "self_harm" : true + } + } ] + } + }, + "grounding" : { + "type" : "document_grounding_service", + "config" : { + "filters" : [ { + "data_repositories" : [ "*" ], + "data_repository_type" : "help.sap.com", + "data_repository_metadata" : [ ], + "document_metadata" : [ ], + "chunk_metadata" : [ ] + } ], + "placeholders" : { + "input" : [ "userMessage" ], + "output" : "groundingContext" + } + } + } + } ] + }, + "placeholder_values" : { }, + "messages_history" : [ ] +} \ No newline at end of file diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java index 805e5fc19..cef2989bd 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OrchestrationController.java @@ -397,4 +397,14 @@ Object configFromRegistry(@RequestParam(value = "format", required = false) fina } return response.getContent(); } + + @GetMapping("/completionWithFallback") + Object completionWithFallback( + @Nullable @RequestParam(value = "format", required = false) final String format) { + final var response = service.completionWithFallback("HelloWorld!"); + if ("json".equals(format)) { + return response; + } + return response.getContent(); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 54e081a8b..b5b20b270 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -16,6 +16,7 @@ import com.sap.ai.sdk.orchestration.ImageItem; import com.sap.ai.sdk.orchestration.LlamaGuardFilter; import com.sap.ai.sdk.orchestration.Message; +import com.sap.ai.sdk.orchestration.OrchestrationAiModel; import com.sap.ai.sdk.orchestration.OrchestrationChatResponse; import com.sap.ai.sdk.orchestration.OrchestrationClient; import com.sap.ai.sdk.orchestration.OrchestrationClientException; @@ -778,4 +779,65 @@ private OrchestrationConfig buildOrchestrationConfig() { .defaults(Map.of("number", "3"))) .model(LLMModelDetails.create().name(GPT_41_NANO.getName()))))); } + + /** + * Chat request to OpenAI through the Orchestration service with a list of modules. If the first + * request fails (which will happen here), the next module is used as a fallback. + * + * @param famousPhrase the phrase to send to the assistant + * @return the assistant response object + */ + @Nonnull + public OrchestrationChatResponse completionWithFallback(@Nonnull final String famousPhrase) { + val prompt = new OrchestrationPrompt(famousPhrase + " Why is this phrase so famous?"); + val workingConfig = + new OrchestrationModuleConfig().withLlmConfig(GPT_4O_MINI.withParam(TEMPERATURE, 0.0)); + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + val secondBrokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name_2", Map.of(), "latest")); + final OrchestrationModuleConfig[] fallbacks = {secondBrokenConfig, workingConfig}; + return client.chatCompletion(prompt, brokenConfig, fallbacks); + } + + /** + * Asynchronous stream of chat request to OpenAI through the Orchestration service with a list of + * modules. If the first request fails (which will happen here), the next module is used as a + * fallback. + * + * @param famousPhrase the phrase to send to the assistant + * @return a stream of assistant message responses + */ + @Nonnull + public Stream streamCompletionWithFallback(@Nonnull final String famousPhrase) { + val prompt = new OrchestrationPrompt(famousPhrase + " Why is this phrase so famous?"); + val workingConfig = + new OrchestrationModuleConfig().withLlmConfig(GPT_4O_MINI.withParam(TEMPERATURE, 0.0)); + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + return client.streamChatCompletion(prompt, brokenConfig, workingConfig); + } + + /** + * Chat request to OpenAI through the Orchestration service with a list of modules. Here, both the + * original and the fallback request fail. + * + * @param famousPhrase the phrase to send to the assistant + * @return the assistant response object + */ + @Nonnull + public OrchestrationChatResponse completionWithFallbackAllFail( + @Nonnull final String famousPhrase) { + val prompt = new OrchestrationPrompt(famousPhrase + " Why is this phrase so famous?"); + val brokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest")); + val secondBrokenConfig = + new OrchestrationModuleConfig() + .withLlmConfig(new OrchestrationAiModel("broken_name_2", Map.of(), "latest")); + return client.chatCompletion(prompt, brokenConfig, secondBrokenConfig); + } } diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 07ed8c0df..e93404c66 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -266,6 +266,18 @@

Orchestration

+
  • +
    + +
    + Chat request with a list of LLM modules to call. If the first fails (which will happen here), the next module is called. +
    +
    +