Fixes for issues with output overrides as well as errors in cooldown, warmup, and rampup logic

Author: markurtz

src/guidellm/__main__.py

@@ -143,9 +143,8 @@ def benchmark():
 )
 @click.option(
     "--rate",
-    type=str,
     callback=cli_tools.parse_list_floats,
-    multiple=False,
+    multiple=True,
     default=BenchmarkGenerativeTextArgs.get_default("rate"),
     help=(
         "Benchmark rate(s) to test. Meaning depends on profile: "
@@ -255,36 +254,46 @@ def benchmark():
 )
 # Output configuration
 @click.option(
-    "--output-path",
-    type=click.Path(),
-    default=BenchmarkGenerativeTextArgs.get_default("output_path"),
-    help=(
-        "Path to save output files. Can be a directory or file. "
-        "If a file, saves that format; mismatched formats save to parent directory."
-    ),
+    "--output-dir",
+    type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
+    default=BenchmarkGenerativeTextArgs.get_default("output_dir"),
+    help="The directory path to save file output types in",
 )
 @click.option(
-    "--output-formats",
+    "--outputs",
+    callback=cli_tools.parse_list,
     multiple=True,
-    type=str,
-    default=BenchmarkGenerativeTextArgs.get_default("output_formats"),
-    help="Output formats for results (e.g., console, json, html, csv).",
+    default=BenchmarkGenerativeTextArgs.get_default("outputs"),
+    help=(
+        "The filename.ext for each of the outputs to create or the "
+        "alises (json, csv, html) for the output files to create with "
+        "their default file names (benchmark.[EXT])"
+    ),
 )
 @click.option(
-    "--disable-console-outputs",
-    is_flag=True,
-    help="Disable console output.",
+    "--output-path",
+    type=click.Path(),
+    default=None,
+    help=(
+        "Legacy parameter for the output path to save the output result to. "
+        "Resolves to fill in output-dir and outputs based on input path."
+    ),
 )
-# Updates configuration
 @click.option(
-    "--disable-progress",
+    "--disable-console",
+    "--disable-console-outputs",  # legacy alias
+    "disable_console",
     is_flag=True,
-    help="Disable progress updates to the console.",
+    help=(
+        "Disable all outputs to the console (updates, interactive progress, results)."
+    ),
 )
 @click.option(
-    "--display-scheduler-stats",
+    "--disable-console-interactive",
+    "--disable-progress",  # legacy alias
+    "disable_console_interactive",
     is_flag=True,
-    help="Display scheduler process statistics.",
+    help="Disable interactive console progress updates.",
 )
 # Aggregators configuration
 @click.option(
@@ -319,17 +328,11 @@ def benchmark():
 )
 @click.option(
     "--rampup",
+    type=float,
     default=BenchmarkGenerativeTextArgs.get_default("rampup"),
-    callback=cli_tools.parse_json,
     help=(
-        "Rampup specification: int, float, or dict as string "
-        "(json or key=value). "
-        "Controls time to linearly ramp up requests. "
-        "Only for Throughput/Concurrent strategies, "
-        "not Synchronous/Rate-based. "
-        "Numeric in (0, 1): percent of duration. "
-        "Numeric >=1: duration in seconds. "
-        "Advanced config: see TransientPhaseConfig schema."
+        "The time, in seconds, to ramp up the request rate over. "
+        "Only applicable for Throughput/Concurrent strategies"
     ),
 )
 @click.option(
@@ -380,26 +383,29 @@ def benchmark():
     help="Maximum global error rate across all benchmarks.",
 )
 def run(**kwargs):
+    # Handle remapping for request params
     request_type = kwargs.pop("request_type", None)
     request_formatter_kwargs = kwargs.pop("request_formatter_kwargs", None)
     kwargs["data_request_formatter"] = (
         request_type
         if not request_formatter_kwargs
         else {"request_type": request_type, **request_formatter_kwargs}
     )
-    kwargs["data"] = cli_tools.format_list_arg(
-        kwargs.get("data"), default=[], simplify_single=False
-    )
-    kwargs["data_args"] = cli_tools.format_list_arg(
-        kwargs.get("data_args"), default=[], simplify_single=False
-    )
-    kwargs["rate"] = cli_tools.format_list_arg(
-        kwargs.get("rate"), default=None, simplify_single=False
-    )
 
-    disable_console_outputs = kwargs.pop("disable_console_outputs", False)
-    display_scheduler_stats = kwargs.pop("display_scheduler_stats", False)
-    disable_progress = kwargs.pop("disable_progress", False)
+    # Handle output path remapping
+    if (output_path := kwargs.pop("output_path", None)) is not None:
+        path = Path(output_path)
+        if path.is_dir():
+            kwargs["output_dir"] = path
+        else:
+            kwargs["output_dir"] = path.parent
+            kwargs["outputs"] = (path.suffix.lstrip(".").lower(),)
+
+    # Handle console options
+    disable_console = kwargs.pop("disable_console", False)
+    disable_console_interactive = (
+        kwargs.pop("disable_console_interactive", False) or disable_console
+    )
 
     try:
         # Only set CLI args that differ from click defaults
@@ -421,13 +427,11 @@ def run(**kwargs):
         benchmark_generative_text(
             args=args,
             progress=(
-                GenerativeConsoleBenchmarkerProgress(
-                    display_scheduler_stats=display_scheduler_stats
-                )
-                if not disable_progress
+                GenerativeConsoleBenchmarkerProgress()
+                if not disable_console_interactive
                 else None
             ),
-            console=Console() if not disable_console_outputs else None,
+            console=Console() if not disable_console else None,
         )
     )
 

src/guidellm/benchmark/entrypoints.py

@@ -21,7 +21,10 @@
 
 from guidellm.backends import Backend, BackendType
 from guidellm.benchmark.benchmarker import Benchmarker
-from guidellm.benchmark.outputs import GenerativeBenchmarkerOutput
+from guidellm.benchmark.outputs import (
+    GenerativeBenchmarkerConsole,
+    GenerativeBenchmarkerOutput,
+)
 from guidellm.benchmark.profiles import Profile, ProfileType
 from guidellm.benchmark.progress import GenerativeConsoleBenchmarkerProgress
 from guidellm.benchmark.schemas import (
@@ -313,7 +316,7 @@ async def resolve_profile(
     profile: StrategyType | ProfileType | Profile,
     rate: list[float] | None,
     random_seed: int,
-    rampup: TransientPhaseConfig,
+    rampup: float,
     constraints: MutableMapping[str, ConstraintInitializer | Any],
     max_seconds: int | float | None,
     max_requests: int | None,
@@ -332,7 +335,8 @@ async def resolve_profile(
     :param profile: Profile type identifier or pre-configured Profile instance
     :param rate: Request rate(s) for the benchmark execution
     :param random_seed: Seed for reproducible random operations
-    :param rampup: Ramp-up phase configuration for the benchmark execution
+    :param warmup: Warm-up phase configuration for the benchmark execution
+        (used for ramp-up duration calculation)
     :param constraints: Dictionary of constraint initializers for benchmark limits
     :param max_seconds: Maximum duration in seconds for the benchmark
     :param max_requests: Maximum number of requests to process
@@ -358,24 +362,21 @@ async def resolve_profile(
     }.items():
         if val is not None:
             constraints[key] = val
-    rampup_duration, _ = rampup.compute_limits(
-        max_requests=max_requests, max_seconds=max_seconds
-    )
 
     if not isinstance(profile, Profile):
         profile = Profile.create(
             rate_type=profile,
             rate=rate,
             random_seed=random_seed,
-            rampup_duration=rampup_duration or 0.0,
+            rampup_duration=rampup,
             constraints={**constraints},
         )
     elif constraints:
         raise ValueError(
             "Constraints must be empty when providing a Profile instance. "
             f"Provided constraints: {constraints} ; provided profile: {profile}"
         )
-    elif rampup_duration is not None:
+    elif rampup > 0.0:
         raise ValueError(
             "Ramp-up duration must not be set when providing a Profile instance. "
             f"Provided rampup: {rampup} ; provided profile: {profile}"
@@ -392,15 +393,15 @@ async def resolve_profile(
 
 
 async def resolve_output_formats(
-    output_formats: OutputFormatT,
-    output_path: str | Path | None,
+    outputs: list[str] | tuple[str],
+    output_dir: str | Path | None,
     console: Console | None = None,
 ) -> dict[str, GenerativeBenchmarkerOutput]:
     """
     Resolve output format specifications into configured output handler instances.
 
-    :param output_formats: Specification of desired output formats
-    :param output_path: Base path for output file generation, or None for default
+    :param outputs: Specification of desired output files/types
+    :param output_dir: Base path for output file generation, or None for default
     :param console: Console instance for progress reporting, or None
     :return: Dictionary mapping format names to configured output handler instances
     """
@@ -409,7 +410,7 @@ async def resolve_output_formats(
     )
 
     resolved = GenerativeBenchmarkerOutput.resolve(
-        output_formats=output_formats, output_path=output_path
+        outputs=outputs, output_dir=output_dir
     )
 
     if console_step:
@@ -473,18 +474,16 @@ async def benchmark_generative_text(
         **(args.dataloader_kwargs or {}),
     )
 
-    rampup = TransientPhaseConfig.create_from_value(args.rampup)
-    rampup.mode = "duration"
     warmup = TransientPhaseConfig.create_from_value(args.warmup)
     cooldown = TransientPhaseConfig.create_from_value(args.cooldown)
     if console:
         console.print_update(
             title="Resolved transient phase configurations",
             details="\n".join(
                 [
-                    f"Rampup: {rampup}",
                     f"Warmup: {warmup}",
                     f"Cooldown: {cooldown}",
+                    f"Rampup (Throughput/Concurrent): {args.rampup}",
                 ]
             ),
             status="success",
@@ -494,7 +493,7 @@ async def benchmark_generative_text(
         profile=args.profile,
         rate=args.rate,
         random_seed=args.random_seed,
-        rampup=rampup,
+        rampup=args.rampup,
         constraints=constraints,
         max_seconds=args.max_seconds,
         max_requests=args.max_requests,
@@ -504,9 +503,7 @@ async def benchmark_generative_text(
         console=console,
     )
     output_formats = await resolve_output_formats(
-        output_formats=args.output_formats,
-        output_path=args.output_path,
-        console=console,
+        outputs=args.outputs, output_dir=args.output_dir, console=console
     )
 
     report = GenerativeBenchmarksReport(args=args)
@@ -541,6 +538,7 @@ async def benchmark_generative_text(
         output_format_results[key] = output_result
 
     if console:
+        await GenerativeBenchmarkerConsole(console=console).finalize(report)
         console.print("\n\n")
         console.print_update(
             title=(
@@ -581,7 +579,9 @@ async def reimport_benchmarks_report(
         )
 
     resolved_output_formats = await resolve_output_formats(
-        output_formats, output_path, console=console
+        output_formats,  # type: ignore[arg-type]
+        output_path,
+        console=console,
     )
     output_format_results = {}
     for key, output in resolved_output_formats.items():