Fixes for non streaming request pathways

Signed-off-by: Mark Kurtz <mark.kurtz@neuralmagic.com>

Author: markurtz

src/guidellm/backends/openai.py

@@ -18,10 +18,7 @@
 import httpx
 
 from guidellm.backends.backend import Backend
-from guidellm.backends.response_handlers import (
-    GenerationResponseHandler,
-    GenerationResponseHandlerFactory,
-)
+from guidellm.backends.response_handlers import GenerationResponseHandlerFactory
 from guidellm.schemas import GenerationRequest, GenerationResponse, RequestInfo
 
 __all__ = ["OpenAIHTTPBackend"]
@@ -244,8 +241,8 @@ async def resolve(  # type: ignore[override]
         )
         request_json = request.arguments.body if not request_files else None
         request_data = request.arguments.body if request_files else None
-        response_handler = self._resolve_response_handler(
-            request_type=request.request_type
+        response_handler = GenerationResponseHandlerFactory.create(
+            request.request_type, handler_overrides=self.response_handlers
         )
 
         if not request.arguments.stream:
@@ -335,20 +332,3 @@ def _resolve_validate_kwargs(
             validate_kwargs["method"] = "GET"
 
         return validate_kwargs
-
-    def _resolve_response_handler(self, request_type: str) -> GenerationResponseHandler:
-        if (
-            self.response_handlers is not None
-            and (handler := self.response_handlers.get(request_type)) is not None
-        ):
-            return handler
-
-        handler_class = GenerationResponseHandlerFactory.get_registered_object(
-            request_type
-        )
-        if handler_class is None:
-            raise ValueError(
-                f"No response handler registered for request type '{request_type}'"
-            )
-
-        return handler_class()

src/guidellm/backends/response_handlers.py

@@ -136,7 +136,8 @@ def compile_non_streaming(
         :return: Standardized GenerationResponse with extracted text and metrics
         """
         choices, usage = self.extract_choices_and_usage(response)
-        text = choices[0].get("text", "") if choices else ""
+        choice = choices[0] if choices else {}
+        text = choice.get("text", "")
         input_metrics, output_metrics = self.extract_metrics(usage, text)
 
         return GenerationResponse(
@@ -164,8 +165,9 @@ def add_streaming_line(self, line: str) -> int | None:
 
         updated = False
         choices, usage = self.extract_choices_and_usage(data)
+        choice = choices[0] if choices else {}
 
-        if choices and (text := choices[0].get("text")):
+        if choices and (text := choice.get("text")):
             self.streaming_texts.append(text)
             updated = True
 
@@ -295,8 +297,8 @@ def compile_non_streaming(
         :return: Standardized GenerationResponse with extracted content and metrics
         """
         choices, usage = self.extract_choices_and_usage(response)
-        choice = choices[0] if choices else {}
-        text = choice.get("content", "")
+        choice: dict[str, dict] = choices[0] if choices else {}
+        text = choice.get("message", {}).get("content", "")
         input_metrics, output_metrics = self.extract_metrics(usage, text)
 
         return GenerationResponse(
@@ -324,8 +326,9 @@ def add_streaming_line(self, line: str) -> int | None:
 
         updated = False
         choices, usage = self.extract_choices_and_usage(data)
+        choice: dict[str, dict] = choices[0] if choices else {}
 
-        if choices and (content := choices[0].get("delta", {}).get("content")):
+        if choices and (content := choice.get("delta", {}).get("content")):
             self.streaming_texts.append(content)
             updated = True
 

src/guidellm/benchmark/schemas/generative/entrypoints.py

@@ -200,7 +200,7 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
         default="generative_column_mapper",
         description="Column mapping preprocessor for dataset fields",
     )
-    data_request_formatter: RequestFormatter | dict[str, str] | str = Field(
+    data_request_formatter: RequestFormatter | dict[str, Any] | str = Field(
         default="chat_completions",
         description="Request formatting preprocessor or template name",
         validation_alias=AliasChoices(

src/guidellm/schemas/request_stats.py

@@ -167,7 +167,10 @@ def time_per_output_token_ms(self) -> float | None:
         """
         if (
             (start := self.info.timings.request_start) is None
-            or (last_token := self.last_token_iteration) is None
+            or (
+                (last_token := self.last_token_iteration or self.request_end_time)
+                is None
+            )
             or (output_tokens := self.output_tokens) is None
             or output_tokens == 0
         ):