feat: [Orchestration] Support Fallback Modules

docs/release_notes.md

@@ -16,7 +16,8 @@
 
 ### 📈 Improvements
 
--[Orchestration] Added new API `OrchestrationTemplateReference#withScope` to support prompt templates with resource-group scope.
+- [Orchestration] Added new API `OrchestrationTemplateReference#withScope` to support prompt templates with resource-group scope.
+- [Orchestration] Chat completion calls now can have multiple module configs to support [fallback modules](https://sap.github.io/ai-sdk/docs/java/orchestration/chat-completion).
 
 ### 🐛 Fixed Issues
 

orchestration/pom.xml

@@ -40,7 +40,7 @@
     <coverage.line>95%</coverage.line>
     <coverage.instruction>93%</coverage.instruction>
     <coverage.branch>74%</coverage.branch>
-    <coverage.method>94%</coverage.method>
+    <coverage.method>95%</coverage.method>
     <coverage.class>100%</coverage.class>
   </properties>
 

orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java

@@ -17,6 +17,9 @@
 import com.sap.ai.sdk.orchestration.model.TranslationModuleConfig;
 import io.vavr.control.Option;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.IntStream;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import lombok.AccessLevel;
@@ -30,24 +33,41 @@
 final class ConfigToRequestTransformer {
   @Nonnull
   static CompletionRequestConfiguration toCompletionPostRequest(
-      @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) {
-    val template = toTemplateModuleConfig(prompt, config.getTemplateConfig());
+      @Nonnull final OrchestrationPrompt prompt,
+      @Nonnull final OrchestrationModuleConfig config,
+      @Nonnull final OrchestrationModuleConfig... fallbackConfigs) {
+
+    final List<OrchestrationModuleConfig> configList = new ArrayList<>();
+    configList.add(config);
+    if (fallbackConfigs.length > 0) {
+      configList.addAll(Arrays.asList(fallbackConfigs));
+    }
+    val templates =
+        configList.stream()
+            .map(conf -> toTemplateModuleConfig(prompt, conf.getTemplateConfig()))
+            .toList();
 
     // note that the config is immutable and implicitly copied here
     // copying is required here, to not alter the original config object, which might be reused for
     // subsequent requests
-    val configCopy = config.withTemplateConfig(template);
+    val configsCopy =
+        IntStream.range(0, configList.size())
+            .mapToObj(i -> configList.get(i).withTemplateConfig(templates.get(i)))
+            .toList();
 
     val messageHistory =
         prompt.getMessagesHistory().stream().map(Message::createChatMessage).toList();
 
-    val moduleConfigs = toModuleConfigs(configCopy);
+    final OrchestrationConfigModules moduleConfigs =
+        configsCopy.size() == 1
+            ? toModuleConfigs(configsCopy.get(0))
+            : toListOfModuleConfigs(configsCopy);
 
-    val reqConfig =
+    val requestConfig =
         OrchestrationConfig.create().modules(moduleConfigs).stream(config.getGlobalStreamOptions());
 
     return CompletionRequestConfiguration.create()
-        .config(reqConfig)
+        .config(requestConfig)
         .placeholderValues(prompt.getTemplateParameters())
         .messagesHistory(messageHistory);
   }
@@ -93,6 +113,18 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig(
 
   @Nonnull
   static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConfig config) {
+    return OrchestrationConfigModules.createInnerModuleConfigs(setupModuleConfigs(config));
+  }
+
+  @Nonnull
+  static OrchestrationConfigModules.ListOfModuleConfigss toListOfModuleConfigs(
+      @Nonnull final List<OrchestrationModuleConfig> configs) {
+    return OrchestrationConfigModules.createListOfModuleConfigss(
+        configs.stream().map(ConfigToRequestTransformer::setupModuleConfigs).toList());
+  }
+
+  @Nonnull
+  private static ModuleConfigs setupModuleConfigs(@Nonnull final OrchestrationModuleConfig config) {
     val llmConfig =
         Option.of(config.getLlmConfig())
             .getOrElseThrow(() -> new IllegalStateException("LLM config is required."));
@@ -117,8 +149,7 @@ static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConf
       inputTranslation.forEach(moduleConfig.getTranslation()::input);
       outputTranslation.forEach(moduleConfig.getTranslation()::output);
     }
-
-    return OrchestrationConfigModules.createInnerModuleConfigs(moduleConfig);
+    return moduleConfig;
   }
 
   @Nonnull

orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java

@@ -14,7 +14,6 @@
 import com.sap.ai.sdk.orchestration.model.EmbeddingsPostRequest;
 import com.sap.ai.sdk.orchestration.model.EmbeddingsPostResponse;
 import com.sap.ai.sdk.orchestration.model.GlobalStreamOptions;
-import com.sap.ai.sdk.orchestration.model.ModuleConfigs;
 import com.sap.ai.sdk.orchestration.model.OrchestrationConfig;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Header;
 import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination;
@@ -74,28 +73,33 @@ public OrchestrationClient(@Nonnull final HttpDestination destination) {
    *
    * @param prompt The {@link OrchestrationPrompt} to generate a completion for.
    * @param config The {@link OrchestrationConfig } configuration to use for the completion.
+   * @param fallbackConfigs Fallback configurations to use.
    * @return The low-level request data object to send to orchestration.
    */
   @Nonnull
   public static CompletionRequestConfiguration toCompletionPostRequest(
-      @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config) {
-    return ConfigToRequestTransformer.toCompletionPostRequest(prompt, config);
+      @Nonnull final OrchestrationPrompt prompt,
+      @Nonnull final OrchestrationModuleConfig config,
+      @Nonnull final OrchestrationModuleConfig... fallbackConfigs) {
+    return ConfigToRequestTransformer.toCompletionPostRequest(prompt, config, fallbackConfigs);
   }
 
   /**
    * Generate a completion for the given prompt.
    *
    * @param prompt The {@link OrchestrationPrompt} to send to orchestration.
-   * @param config The {@link ModuleConfigs} configuration to use for the completion.
+   * @param config the configuration to use
+   * @param fallbackConfigs fallback configurations
    * @return the completion output
    * @throws OrchestrationClientException if the request fails.
    */
   @Nonnull
   public OrchestrationChatResponse chatCompletion(
-      @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config)
+      @Nonnull final OrchestrationPrompt prompt,
+      @Nonnull final OrchestrationModuleConfig config,
+      @Nonnull final OrchestrationModuleConfig... fallbackConfigs)
       throws OrchestrationClientException {
-
-    val request = toCompletionPostRequest(prompt, config);
+    val request = toCompletionPostRequest(prompt, config, fallbackConfigs);
     val response = executeRequest(request);
     return new OrchestrationChatResponse(response);
   }
@@ -105,17 +109,20 @@ public OrchestrationChatResponse chatCompletion(
    *
    * @param prompt a text message.
    * @param config the configuration to use
+   * @param fallbackConfigs fallback configurations
    * @return a stream of message deltas
    * @throws OrchestrationClientException if the request fails or if the finish reason is
    *     content_filter
    * @since 1.1.0
    */
   @Nonnull
   public Stream<String> streamChatCompletion(
-      @Nonnull final OrchestrationPrompt prompt, @Nonnull final OrchestrationModuleConfig config)
+      @Nonnull final OrchestrationPrompt prompt,
+      @Nonnull final OrchestrationModuleConfig config,
+      @Nonnull final OrchestrationModuleConfig... fallbackConfigs)
       throws OrchestrationClientException {
 
-    val request = toCompletionPostRequest(prompt, config);
+    val request = toCompletionPostRequest(prompt, config, fallbackConfigs);
     return streamChatCompletionDeltas(request)
         .peek(OrchestrationClient::throwOnContentFilter)
         .map(OrchestrationChatCompletionDelta::getDeltaContent);

orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java

@@ -1504,4 +1504,45 @@ void testGetAllMessages() {
     assertThat(messageListTools.get(1)).isInstanceOf(AssistantMessage.class);
     assertThat(messageListTools.get(2)).isInstanceOf(ToolMessage.class);
   }
+
+  @Test
+  void testFallbackModules() throws IOException {
+    stubFor(
+        post(urlPathEqualTo("/v2/completion"))
+            .willReturn(
+                aResponse()
+                    .withBodyFile("fallbackResponse.json")
+                    .withHeader("Content-Type", "application/json")));
+
+    final var prompt = new OrchestrationPrompt("HelloWorld! Why is this phrase so famous?");
+    final var llamaFilter = new LlamaGuardFilter().config(LlamaGuard38b.create().selfHarm(true));
+    val groundingConfig =
+        Grounding.create()
+            .filters(
+                DocumentGroundingFilter.create()
+                    .dataRepositoryType(DataRepositoryType.HELP_SAP_COM));
+
+    final var workingConfig =
+        new OrchestrationModuleConfig()
+            .withLlmConfig(GPT_4O_MINI.withParam(TEMPERATURE, 0.0))
+            .withInputFiltering(llamaFilter)
+            .withGrounding(groundingConfig);
+    final var brokenConfig =
+        workingConfig.withLlmConfig(new OrchestrationAiModel("broken_name", Map.of(), "latest"));
+
+    final var response = client.chatCompletion(prompt, brokenConfig, workingConfig);
+
+    var intermediateFailure = response.getOriginalResponse().getIntermediateFailures().get(0);
+    assertThat(intermediateFailure.getCode()).isEqualTo(400);
+    assertThat(intermediateFailure.getLocation()).isEqualTo("Request Body");
+    assertThat(intermediateFailure.getRequestId())
+        .isEqualTo("a562703b-7fe5-97ba-b417-e8140a25fb7c");
+    assertThat(intermediateFailure.getMessage())
+        .isEqualTo("400 - Request Body: Model broken_name not supported.");
+    assertThat(intermediateFailure.getHeaders().get("Content-Type")).isEqualTo("application/json");
+
+    final String request = fileLoaderStr.apply("fallbackRequest.json");
+    verify(
+        postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request)));
+  }
 }

orchestration/src/test/resources/__files/fallbackResponse.json

@@ -0,0 +1,81 @@
+{
+  "request_id": "a562703b-7fe5-97ba-b417-e8140a25fb7c",
+  "intermediate_results": {
+    "templating": [
+      {
+        "content": "HelloWorld! Why is this phrase so famous?",
+        "role": "user"
+      }
+    ],
+    "llm": {
+      "id": "",
+      "object": "chat.completion",
+      "created": 1769700208,
+      "model": "gemini-2.5-flash",
+      "choices": [
+        {
+          "index": 0,
+          "message": {
+            "role": "assistant",
+            "content": "\"Hello, World!\" is arguably the most famous phrase in the world of programming, and its fame stems from a combination of historical significance, pedagogical utility, and cultural impact.\n\nHere's why it's so famous:\n\n1.  **Historical Origin and Standardization:**\n    *   While simple output programs existed before, the phrase \"Hello, World!\" was popularized and standardized by **Brian Kernighan and Dennis Ritchie** in their seminal 1978 book, **\"The C Programming Language.\"**\n    *   In the book, the very first example program printed \"hello, world\" (lowercase, no exclamation mark initially). This book became the definitive guide for C, and C became one of the most influential programming languages ever. As C spread, so did its iconic first program.\n    *   Before K&R, a similar example (\"hello\") appeared in a 1974 Bell Labs internal memo by Kernighan, demonstrating the BCPL language.\n\n2.  **Pedagogical Value (Teaching Tool):**\n    *   **First Step for Beginners:** It's the absolute simplest program that actually *does* something visible. For someone learning to code, successfully running \"Hello, World!\" is an immediate, tangible victory.\n    *   **Low Barrier to Entry:** It requires minimal syntax and no complex concepts (like variables, loops, functions, or data structures). It focuses solely on the core idea of output.\n    *   **Immediate Gratification:** Seeing \"Hello, World!\" appear on the screen provides instant feedback and motivation, confirming that the development environment is set up correctly and the basic compilation/execution process works.\n    *   **Introduction to Core Concepts:** It subtly introduces fundamental ideas like:\n        *   The concept of a program.\n        *   The idea of output (printing to the console).\n        *   The basic structure of a program in a given language.\n\n3.  **Practical Utility (Testing and Verification):**\n    *   **Sanity Check:** When setting up a new programming language, a new development environment (IDE), a new compiler, or a new operating system, \"Hello, World!\" is the go-to program to verify that everything is working correctly. If \"Hello, World!\" doesn't run, you know the problem is fundamental to your setup, not your complex code.\n    *   **Minimal Viable Product (MVP) for Setup:** It's the quickest way to confirm that the toolchain (editor, compiler/interpreter, linker, runtime) is functional.\n\n4.  **Cultural Significance and Rite of Passage:**\n    *   **Universal Rite of Passage:** Every programmer, regardless of the language they learn, almost certainly starts with \"Hello, World!\" It's a shared experience that connects developers across generations and technologies.\n    *   **Symbol of Beginning:** It represents the very first step into the vast and complex world of programming. It's a literal \"hello\" to the machine and the coding journey.\n    *   **Ubiquity:** Its presence in countless tutorials, textbooks, and online guides for virtually every programming language has cemented its status as an iconic and universally recognized phrase.\n\nIn essence, \"Hello, World!\" is famous because it's simple, effective, historically significant, and serves as a universal welcome mat for anyone stepping into the world of computer programming.",
+            "tool_calls": []
+          },
+          "finish_reason": "stop"
+        }
+      ],
+      "usage": {
+        "completion_tokens": 1728,
+        "prompt_tokens": 9,
+        "total_tokens": 1737,
+        "prompt_tokens_details": {
+          "cached_tokens": 0
+        },
+        "completion_tokens_details": {
+          "reasoning_tokens": 1024
+        }
+      },
+      "citations": []
+    },
+    "output_unmasking": []
+  },
+  "final_result": {
+    "id": "",
+    "object": "chat.completion",
+    "created": 1769700208,
+    "model": "gemini-2.5-flash",
+    "choices": [
+      {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": "\"Hello, World!\" is arguably the most famous phrase in the world of programming, and its fame stems from a combination of historical significance, pedagogical utility, and cultural impact.\n\nHere's why it's so famous:\n\n1.  **Historical Origin and Standardization:**\n    *   While simple output programs existed before, the phrase \"Hello, World!\" was popularized and standardized by **Brian Kernighan and Dennis Ritchie** in their seminal 1978 book, **\"The C Programming Language.\"**\n    *   In the book, the very first example program printed \"hello, world\" (lowercase, no exclamation mark initially). This book became the definitive guide for C, and C became one of the most influential programming languages ever. As C spread, so did its iconic first program.\n    *   Before K&R, a similar example (\"hello\") appeared in a 1974 Bell Labs internal memo by Kernighan, demonstrating the BCPL language.\n\n2.  **Pedagogical Value (Teaching Tool):**\n    *   **First Step for Beginners:** It's the absolute simplest program that actually *does* something visible. For someone learning to code, successfully running \"Hello, World!\" is an immediate, tangible victory.\n    *   **Low Barrier to Entry:** It requires minimal syntax and no complex concepts (like variables, loops, functions, or data structures). It focuses solely on the core idea of output.\n    *   **Immediate Gratification:** Seeing \"Hello, World!\" appear on the screen provides instant feedback and motivation, confirming that the development environment is set up correctly and the basic compilation/execution process works.\n    *   **Introduction to Core Concepts:** It subtly introduces fundamental ideas like:\n        *   The concept of a program.\n        *   The idea of output (printing to the console).\n        *   The basic structure of a program in a given language.\n\n3.  **Practical Utility (Testing and Verification):**\n    *   **Sanity Check:** When setting up a new programming language, a new development environment (IDE), a new compiler, or a new operating system, \"Hello, World!\" is the go-to program to verify that everything is working correctly. If \"Hello, World!\" doesn't run, you know the problem is fundamental to your setup, not your complex code.\n    *   **Minimal Viable Product (MVP) for Setup:** It's the quickest way to confirm that the toolchain (editor, compiler/interpreter, linker, runtime) is functional.\n\n4.  **Cultural Significance and Rite of Passage:**\n    *   **Universal Rite of Passage:** Every programmer, regardless of the language they learn, almost certainly starts with \"Hello, World!\" It's a shared experience that connects developers across generations and technologies.\n    *   **Symbol of Beginning:** It represents the very first step into the vast and complex world of programming. It's a literal \"hello\" to the machine and the coding journey.\n    *   **Ubiquity:** Its presence in countless tutorials, textbooks, and online guides for virtually every programming language has cemented its status as an iconic and universally recognized phrase.\n\nIn essence, \"Hello, World!\" is famous because it's simple, effective, historically significant, and serves as a universal welcome mat for anyone stepping into the world of computer programming.",
+          "tool_calls": []
+        },
+        "finish_reason": "stop"
+      }
+    ],
+    "usage": {
+      "completion_tokens": 1728,
+      "prompt_tokens": 9,
+      "total_tokens": 1737,
+      "prompt_tokens_details": {
+        "cached_tokens": 0
+      },
+      "completion_tokens_details": {
+        "reasoning_tokens": 1024
+      }
+    },
+    "citations": []
+  },
+  "intermediate_failures": [
+    {
+      "request_id": "a562703b-7fe5-97ba-b417-e8140a25fb7c",
+      "code": 400,
+      "message": "400 - Request Body: Model broken_name not supported.",
+      "location": "Request Body",
+      "headers": {
+        "Content-Type": "application/json"
+      }
+    }
+  ]
+}