diff --git a/.github/workflows/test-h3-video.yml b/.github/workflows/test-h3-video.yml index 4df2172014..3bf046f3e3 100644 --- a/.github/workflows/test-h3-video.yml +++ b/.github/workflows/test-h3-video.yml @@ -41,7 +41,6 @@ concurrency: jobs: test: name: H3 CPU harness and workflow checks - if: ${{ github.event.pull_request.draft != true }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -58,7 +57,9 @@ jobs: check_env_vars PYTHONPATH uv run --no-project --python 3.12 \ --with 'av==16.1.0' --with 'numpy==2.3.5' \ - --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q + --with 'pyyaml==6.0.3' \ + --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' \ + --with 'rfc3339-validator==0.1.4' python -m pytest -q - name: Check entry shell syntax run: | bash -n experimental/video-generation/runtime-entry.example.sh @@ -76,7 +77,6 @@ jobs: staged-power-python310: name: H3 staged power on Python 3.10 - if: ${{ github.event.pull_request.draft != true }} runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/experimental/video-generation/README.md b/experimental/video-generation/README.md index e5899d28ce..f86c420182 100644 --- a/experimental/video-generation/README.md +++ b/experimental/video-generation/README.md @@ -2,6 +2,71 @@ **English** | [中文](README_zh.md) +## Wan2.2 local runner integration + +The client accepts `Wan-AI/Wan2.2-T2V-A14B-Diffusers` on SGLang and vLLM-Omni; +the supervised `serving-smoke` path starts SGLang. A pinned eight-H100 +vLLM-Omni server completed one Wan request. The full workload, matched performance, +quality, and production delivery remain unverified. + +The [Wan plan](mvp/wan22-serving.plan.json) freezes checkpoint +`5be7df9619b54f4e2667b2755bc6a756675b5cd7`, four video-only prompts, one warmup, +832×480, 81 frames at 16 FPS (5.0625 seconds), 40 steps, high/low expert guidance +4/3, flow shift 12, and an explicitly empty negative prompt. This is a new workload, +not a matched H3 comparison or the upstream default negative prompt. The protocol +was inspected at SGLang `71de97b264b04dcd514cf904003028aefe9775c8`. That runtime +ignores per-request Wan flow-shift overrides, so only 12 is admitted; high-stage +guidance must exceed 1 for the runtime to apply both expert guidance values. +The [shared Wan spec](specs/wan.yaml) retains that four-case plan as `frozen-3733` +and defines `shared-v1` with two seeds per prompt. The [serving note](specs/wan-vllm-omni-serving.md) +records the pinned vLLM-Omni setup and single-request evidence. The spec loader +is shared with H3 and keeps the plan hash separate from source citations. + +Preview requests without a server or GPU, from `experimental/video-generation`: + +```bash +python -m evaluator.cli run mvp/wan22-serving.plan.json --runtime sglang +``` + +The [GPU template](mvp/wan22-gpu-job.template.json) retains unapproved authorization, +unresolved paths and an empty file manifest. Before separately authorized execution, +bind it to task-owned SemiAnalysis resources and the compatible prepared runtime, +verify all checkpoint files, and set the actual budget. The local supervised entry is +`python -m evaluator.cli serving-smoke --spec --output --concurrencies 1`. +Its first cell has four measured requests plus one warmup. The template does not +establish one-GPU memory fit. + +Initial support is NVIDIA, TP1 and pure Ulysses over 1/2/4/8 GPUs, with explicit +DiT/text-encoder/VAE offload booleans and layerwise offload disabled. The launcher +disables runtime warmup, CFG parallelism and Ring, and uses a job-owned persistent +output directory so the polled `/v1/videos/{id}/content` route can deliver the MP4. +The harness records its own warmup. H3-only launch flags/timing patches, AMD, +and paired quality qualification are rejected for this Wan launcher. The runner +also accepts an operator-managed vLLM-Omni endpoint using the audited multipart transport. +The policy records only an uncalibrated identity; it contains no audio thresholds. + +Media validation requires exact geometry, cadence, frame count and no audio stream. +Audio generation and quality qualification are unsupported; no audio/A-V score is +invented. Missing quality and quality/SLO goodput stay null in the dashboard. +Timeouts retain unknown remote completion and stop further submissions; deleting a +job in this pinned server does not prove cancellation. + +The local supervised entry writes a `video_serving_smoke_matrix` and retains failed +and unstarted request counts. The operator-managed `run --execute` entry writes a +separate run receipt. The parent H3 CI now launches fresh jobs through native +srt-slurm video matrix rows. Wan has no native recipe or config row, so these local +entries do not submit or qualify a Wan CI job. A native Wan row needs its own +reviewed recipe and artifact wiring. The paired H3 `result.json` exporter is not +used for Wan. The app accepts the canonical Diffusers ID and the earlier short +display ID, and rejects conflicting model declarations. + +CPU checks cover both HTTP transports with encoded fixtures, full video-only +decoding, launch previews, failure seals and dashboard parsing. The retained H100 +record proves one request completed; it does not qualify the shared workload or a +production path. Fixtures are never included as measured results. + +## Existing H3 lane + This experimental lane runs the existing H3 supervisor inside InferenceX CI on prepared SemiAnalysis NVIDIA resources. Its first target is a bounded same-build smoke: original generated MP4s, full video/audio validation, measured requests, and @@ -250,7 +315,8 @@ Regression mode additionally requires the existing calibrated acceptance gate. cd experimental/video-generation PYTHONPATH=../../inferencex-e2e uv run --no-project --python 3.12 \ --with 'av==16.1.0' --with 'numpy==2.3.5' \ - --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q + --with 'pyyaml==6.0.3' --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' \ + --with 'rfc3339-validator==0.1.4' python -m pytest -q bash -n runtime-entry.example.sh ``` diff --git a/experimental/video-generation/README_zh.md b/experimental/video-generation/README_zh.md index 844acc75c9..4ae7526fe8 100644 --- a/experimental/video-generation/README_zh.md +++ b/experimental/video-generation/README_zh.md @@ -2,6 +2,61 @@ [English](README.md) | **中文** +## Wan2.2 本地 runner 接入 + +现有 client 可通过 SGLang 或 vLLM-Omni 运行 +`Wan-AI/Wan2.2-T2V-A14B-Diffusers`;受控 `serving-smoke` 路径启动 SGLang。 +一台固定版本的八卡 H100 vLLM-Omni +服务完成了一次 Wan 请求。完整工作负载、匹配的性能对比、质量及生产交付仍未验证。 + +[Wan plan](mvp/wan22-serving.plan.json) 固定 checkpoint +`5be7df9619b54f4e2667b2755bc6a756675b5cd7`,四条纯视频提示词、一次独立 warmup, +832×480、81 帧、16 FPS(5.0625 秒)、40 步、两阶段 guidance 4/3、flow shift 12, +并显式使用空 negative prompt。这是新工作负载,不是与 H3 匹配的对照,也不是上游 +默认 negative prompt。协议按 SGLang `71de97b264b04dcd514cf904003028aefe9775c8` +核对。该版本忽略请求级 Wan flow-shift 覆盖,因此仅接受 12;第一阶段 guidance 必须 +大于 1,运行时才会应用两个 expert 的 guidance 值。 +[共享 Wan spec](specs/wan.yaml) 用 `frozen-3733` 保留原来的四个 case,并在 +`shared-v1` 中为每条提示词加入第二个 seed。[服务记录](specs/wan-vllm-omni-serving.md) +保留固定版本的 vLLM-Omni 配置和单次请求证据。spec loader 与 H3 共用,plan hash +不包含来源引用。 + +在 `experimental/video-generation` 下无 GPU、无服务地预览请求: + +```bash +python -m evaluator.cli run mvp/wan22-serving.plan.json --runtime sglang +``` + +[GPU 模板](mvp/wan22-gpu-job.template.json) 保持未批准授权、未填写路径和空文件清单。 +另行授权实际运行前,需要绑定任务拥有的 SemiAnalysis 资源及兼容的已有 runtime, +核实全部 checkpoint 文件,并设置实际预算。本地受控入口是 +`python -m evaluator.cli serving-smoke --spec --output --concurrencies 1`。 +第一格为四条测量请求加一次 warmup;模板不能证明单卡显存足够。 + +初始支持 NVIDIA、TP1、1/2/4/8 卡纯 Ulysses,显式设置 DiT/text-encoder/VAE +offload,禁用 layerwise offload。launcher 关闭运行时 warmup、CFG 并行和 Ring, +使用本次任务拥有的持久输出目录,确保轮询后的 `/v1/videos/{id}/content` 能下载 MP4。 +harness 单独记录 warmup。该 Wan launcher 拒绝 H3 专用启动参数/计时补丁、AMD +及配对质量准入。runner 也可通过已核对的 multipart 协议连接由操作员管理的 +vLLM-Omni 服务。policy 仅记录未校准身份,不含音频阈值。 + +媒体验证要求分辨率、时间间隔、帧数符合计划,且无音频流。音频生成和质量准入为 +unsupported,不生成音频或 A/V 分数。Dashboard 缺失质量及质量/SLO goodput 仍为 null。 +超时后保留远端是否完成的未知状态并停止后续提交;该版本删除 job 不代表真正取消。 + +本地受控入口写入 `video_serving_smoke_matrix`,保留失败及未启动的请求数。 +由操作员管理的 `run --execute` 入口另写一份运行收据。父分支的 H3 CI 现在通过原生 +srt-slurm video matrix 行启动新作业。Wan 尚无对应的原生 recipe 或 config 行, +因此这些本地入口不会提交或验收 Wan CI 作业。原生 Wan 行仍需经审核的 recipe +及产物接线。Wan 不使用 H3 配对 `result.json` exporter。App 接受规范 Diffusers ID +与此前短显示 ID,并拒绝冲突的模型声明。 + +CPU 检查覆盖两种使用编码 fixture 的 HTTP transport、纯视频完整解码、启动预览、 +失败封存及 Dashboard 解析。保留的 H100 记录只证明一次请求已完成,不能代表共享 +工作负载或生产路径通过验收。fixture 不进入实测结果。 + +## 既有 H3 路径 + 此实验任务在 InferenceX CI 内使用 SemiAnalysis H200 资源运行现有 H3 supervisor。首个目标是有时间上限的同版本冒烟测试:保留实际生成的 MP4, 完整验证视频和音频,记录请求耗时,并验证清理结果。此任务不写入原生 @@ -135,9 +190,9 @@ H200 硬件盘点、CPU 导出)与 `h3-fidelity.yml`(保留媒体比较) ## 结果与本地检查 -成功执行还会发布[前端结果契约](RESULTS_zh.md):版本化的 `result.json`、 +成功执行还会发布[前端结果契约(英文)](RESULTS.md):版本化的 `result.json`、 逐卡功率序列、分阶段能量与覆盖率,以及可离线打开的 `power-report.html`。 -`h3-results--` 包含索引、JSON Schema、双语指标说明,以及每次 +`h3-results--` 包含索引、JSON Schema、指标说明,以及每次 原始执行的媒体、测量、日志和报告。原始执行身份与校验文件和本次导出身份分开保留。 遥测无效时功率和能量为空;导出失败会保留错误日志并返回失败状态。 @@ -170,7 +225,7 @@ gh workflow run h3-video.yml --repo SemiAnalysisAI/InferenceX \ 配置、实际执行、默认及最大功率上限均为 700 W;历史生成时的设置仍未知。 原始盘点记录中的 unknown 分类保持不变;导出器使用新 producer 提交解析原始 XML,同时支持带或不带 `0x` 的 PCI 编号,不重新查询硬件。 -[已保留的 A/A 测量及限制](RESULTS_zh.md#已观测的-aa-证据)提供具体数据。 +[已保留的 A/A 测量及限制(英文)](RESULTS.md#observed-aa-evidence)提供具体数据。 原始工作负载失败时仍上传失败证据,但不会启动成功结果导出。 @@ -189,7 +244,8 @@ serving 运行由其基准测试作业上传 `h3-results--`; cd experimental/video-generation PYTHONPATH=../../inferencex-e2e uv run --no-project --python 3.12 \ --with 'av==16.1.0' --with 'numpy==2.3.5' \ - --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' python -m pytest -q + --with 'pyyaml==6.0.3' --with 'pytest>=8,<9' --with 'jsonschema>=4,<5' \ + --with 'rfc3339-validator==0.1.4' python -m pytest -q bash -n runtime-entry.example.sh ``` diff --git a/experimental/video-generation/RESULTS.md b/experimental/video-generation/RESULTS.md index 1379ce01c8..66a07235e4 100644 --- a/experimental/video-generation/RESULTS.md +++ b/experimental/video-generation/RESULTS.md @@ -1,7 +1,5 @@ # H3 backend result contract -**English** | [中文](./RESULTS_zh.md) - `result.json` is the frontend entry point for a downloaded H3 CI artifact. Its `schema_version` is **1.0.0**; [result.schema.json](./result.schema.json) describes its external shape. Reject unknown versions. This contract does not ingest a @@ -200,3 +198,83 @@ Each duration is a separate frozen workload. Do not pool these durations or interpret their latency difference as a regression. One measured clip per role provides point estimates only; workload execution passed while regression remains uncalibrated and inconclusive. + +## Offline deployment-frontier replay + +`evaluator.mvp_deployment_frontier` selects retained, normalized dashboard +observations for a local comparison; it does not collect new measurements or +qualify an artifact. This is the backend counterpart of the video-only wrapper in +[App #1193](https://github.com/SemiAnalysisAI/InferenceX-app/pull/1193), delivered +with [Backend #2916](https://github.com/SemiAnalysisAI/InferenceX/pull/2916). +The shared LLM/calculator Pareto implementation is unchanged. + +Pass a JSON object with `points`. Each point supplies `id`, finite numeric `x` and +`y`, `hardwareKey`, canonical `cohortKey` (workload plus generation settings), +canonical `deploymentKey` (including replicas, parallel layout, offload, batching +and engine/runtime), and the producer's `queueingStatus` (`unqueued`, `queueing`, +or `unknown`). Optional `hardwareHealth.status: "fail"` excludes a point even +when dominated deployments are requested. Missing canonical cohorts remain +isolated points. Unknown request capacity and missing axes cannot rank. Exact +coordinate ties retain distinct measured identities; a repeated observation of +one deployment does not create a connecting deployment line. No interpolation +or extrapolation is performed. + +With `--quality-required`, the reader checks the source-bound `quality` assessment; +an opaque `qualifiedQualityCohort` string cannot grant admission. The assessment +must declare `scale: "ordinal_0_to_4"` and preserve all seven original rubric IDs: +`prompt_adherence`, `visual_fidelity`, `temporal_consistency`, `motion_plausibility`, +`audio_quality`, `audio_content`, and `av_sync`. Zero is a real score, not missing +information. Legacy 1–5 scores or renamed dimensions are not silently converted. +The registry remains `human.absolute_dimension_rating`, version `0.2.0-draft`. + +Every critical dimension must have a passing decision, complete target-clip +coverage, identified evaluator, and a calibrated frozen rule with provenance. +`--quality-metric` selects the displayed dimension (default `prompt_adherence`); +`--quality-threshold` can only tighten that dimension's frozen threshold. Other +dimensions still require their own frozen rules. The derived quality cohort includes +all seven evaluator and calibration identities, not only the selected dimension. +Different workload/generation or quality cohorts never dominate each other or share +a connecting line. These checks validate recorded contract fields; they do not +establish that human judgments or calibration are scientifically valid, or turn +media integrity/fidelity into perceptual quality. + +```bash +cd experimental/video-generation +python -m evaluator.mvp_deployment_frontier normalized-observations.json \ + --output frontier.json --x-better lower --y-better higher +``` + +The JSON output retains input identities, selected observations, each separate +frontier and exclusion reasons. `--include-dominated` matches the chart's Optimal +Only off state; `--quality-required` applies the qualification filter first. +`tests/fixtures/deployment-frontier.json` contains shared synthetic direction/tie +cases for the backend and App tests, never measured benchmark evidence. + +## Optional source-bound dashboard observations + +[`dashboard-observations.schema.json`](./dashboard-observations.schema.json) +defines an additive `dashboard-observations.json` sidecar, independent of +`result.json` version 1.0.0. Its root contains `schemaVersion: 1`, `sourceRunId`, +`sourceSha`, and a nonempty `cells` map. Each known cell binds `runSha256` and +`specSha256` to that cell's existing measured run and specification; optional +`deployment`, `hardwareHealth`, and `quality` objects preserve unknowns as null. +The sidecar's SHA256 must appear in both `SHA256SUMS` and +`manifest.evidence["dashboard-observations.json"]`. Ingestion verifies the raw +bytes, source run/commit, cell existence and run/spec hashes before exposing any +of its fields. Schema validation alone cannot establish those relationships. + +Deployment fields distinguish observed `batchSize` from configured +`maxBatchSize`, and retain Ring/CFG, individual offload flags, precision scope, +engine and generation identity. A `hardwareHealth` record carries its status, +reason and evidence; incomplete monitoring remains unknown. Quality stores +the required `ordinal_0_to_4` scale and separate original rubric dimensions with +contract/rubric/evaluator identities and hashes, sample coverage and a frozen +calibration record. Nullable unjudged data is valid +contract data, but cannot qualify a frontier. A syntactically valid `pass` is +also insufficient without all seven calibrated protocol checks in the App and +backend frontier reader. + +The existing exporter does not create this optional sidecar automatically. +Original archives and source seals remain immutable. Explicit local curator +enrichment can exercise these fields for a preview, but must be labelled as +unsealed replay data; it is not equivalent to this sealed ingestion contract. diff --git a/experimental/video-generation/RESULTS_zh.md b/experimental/video-generation/RESULTS_zh.md deleted file mode 100644 index 88e233a356..0000000000 --- a/experimental/video-generation/RESULTS_zh.md +++ /dev/null @@ -1,183 +0,0 @@ -# H3 后端结果契约 - -[English](./RESULTS.md) | **中文** - -`result.json` 是下载后的 H3 CI 产物面向前端的入口。其 `schema_version` -为 **1.0.0**,[result.schema.json](./result.schema.json) 定义对外结构。 -消费者必须拒绝未知版本。此契约不负责写入 InferenceX 数据库,也不代表发布验收。 - -产物保留原始媒体、请求记录、遥测、运行时及客户端日志、模型与运行时身份、 -Slurm 回执,以及原有便携报告 `report/index.html`。导出器新增 `result.json`、 -带时间戳的 `power/baseline.json`、`power/candidate.json` 和 `power-report.html`。 -原文件不会被重写。发布方在导出后更新 `SHA256SUMS`;重处理时另存原始校验文件。 -媒体及报告路径相对产物根目录,并附 SHA256。打开报告前需解压整个产物。 - -## 读取结果 - -1. 先检查 `status` 和 `invalid_reasons`。无效产物会写入失败结果并抛出错误, - 让 CI 保留日志,同时使导出任务失败。 -2. 分开读取 `workload_status` 与 `regression_status`。A/A 视频有效且负载执行 - 成功时,未校准策略仍可令回归结论为 `inconclusive`。此 MVP 的 - `release_qualified` 始终为 false。 -3. 检查 `roles..metrics.status` 和各功率阶段的 `valid`。 - 导出完成不代表所有功率数据都有效。无效功率及能量为 null,不能当作零。 - 一个阶段通过不能使另一个阶段有效。 -4. `execution.ci` 标识**原始 GPU 执行**,包括提交、运行及尝试编号。 - `producer` 标识导出器提交、当前 CI 和源码哈希。重处理不会产生新的 GPU 测量。 -5. `hardware.selected_gpu_count` 是实际测量设备数;`reserved_gpu_count` 来自 - Slurm `AllocTRES`。保留八张卡、使用四张卡时,功率只覆盖四张卡,计算额度 - 则按八张卡的保留时间计费。 - -`workload.plan` 固定提示词、随机种子、视频尺寸/时长/音频格式、步数、重复次数 -和预热次数,`workload.server` 保留运行时设置。`execution` 关联 CI、Slurm、 -GPU UUID、模型清单及实测运行时源码。导出器检查现有完整校验清单,拒绝不安全 -路径及符号链接,复用 `verify_measurement_job`,并检查原报告的本地引用。 -这是对可信运行器记录和媒体哈希的验证;导出器不会重新解码媒体,也不提供独立 -硬件认证。此前完整解码分析仍绑定原始媒体字节。若提供独立获取的 GitHub 元数据, -必须核对成功的 H3 作业、运行编号、尝试编号、URL 及执行提交。同次运行导出时, -只有与导出器运行、尝试、仓库及提交完全一致的工作流才允许仍为 `in_progress`; -`workflow_status_at_export` 保留此状态,同时仍核对已经完成的 H3 作业和下载产物。 - -## 指标定义 - -| 指标 | 单位与边界 | 有效性及限制 | -| --- | --- | --- | -| 请求延迟 | 从提交到下载完成并通过技术校验的秒数 | 只统计有效测量视频,不含启动和预热;保留终态、下载及校验分段时间。 | -| 有效视频数/秒 | 有效测量视频数 / 记录的测量区间墙钟秒数 | 包含失败尝试。串行模式包含校验;服务模式截止于最后一次交付或传输失败。不代表可持续容量。 | -| 服务交付延迟 | 提交到视频下载完成的秒数;P50/P90/P95 | 只统计技术校验有效的测量视频。P90/P95 至少需要 10/20 个样本;此门槛不代表统计资格。失败仍保留在结果计数中。 | -| 截止时间内的有效吞吐 | 技术校验有效且按时交付的视频数 / 服务测量区间秒数 | 截止时间由操作者设置。达标比例的分母为所有计划测量请求,包括失败和未启动请求。 | -| 完成计数 | 计划、尝试、完成、有效、失败、未启动的视频数 | 完成的视频仍可能技术校验失败;预热记录独立保留。 | -| GPU 显存 | 每个选中 UUID 的设备已用 MiB 观测峰值 | 保留原有整个角色及含预热客户端区间,不是精确分配器峰值。 | -| 技术完整性 | 完整视频/音频解码检查,各检查自带单位 | 覆盖解码、几何、时长、时间戳/帧间隔、运动及声音缺陷;不代表语义或感知质量。 | -| 配对输出保真度 | 视频 PSNR dB、音频频谱余弦及 RMS 绝对比例误差 | 比较原始解码输出;视频完全一致时,有限 PSNR 为 null,`exact_match=true`。 | -| GPU 功率 | 每卡带时间戳的 W,及选中 GPU 的功率总和 | 板级传感器读数含设备显存,不含主机及未选中的 GPU。 | -| 平均/观测峰值功率 | 积分能量 J / 阶段秒数;区间内传感器 W 最大值 | 平均值按时间加权;峰值是采样观测值,不是瞬时电气峰值。 | -| 每个有效视频的 GPU 能量 | 梯形积分 J / 技术校验有效的测量视频数 | 串行请求区间不含下载/解码;服务模式对首次提交到最后观测终态的整个区间积分,包含期间的空闲、下载及校验时间。覆盖无效或有效视频数为零时为 null。 | - -每个功率文件具有版本化的 `sample_series`、`windows`、`phases`、`semantics` -和 `clock_alignment`。分别记录启动、每次预热,以及每次测量从提交到观测到 -提供方终态的区间。区间保留时钟来源及不确定性、精确单调时钟边界、每卡采样数 -和间隔、覆盖秒数/比例、边界包围情况及无效原因。只有所有必需区间有效时,才 -汇总该阶段。即使派生指标被保留为空,完整时间序列仍可下载。 - -服务模式把重叠的测量请求区间合并为一个完整区间,板级能量只积分一次,不归因 -到单个请求。启动和预热仍然独立。该生成区间与吞吐的交付区间不同,后者还包含 -最后一次下载或传输失败。 - -可选的 `roles..metrics.serving` 保留原始交付延迟样本、结果计数、观测提交 -速率、客户端同时在途请求峰值、每秒生成视频秒数,以及截止时间内的有效吞吐。 -`records` 保留任务 ID 和客户端生命周期时间。交付区间不等待本地校验结束,但 -两者可以重叠;延迟包含轮询开销,不代表服务端排队时间。 -`execution.deployment` 记录单个端点、选中的 GPU 和服务设置;预留 GPU 仍在 -`hardware` 中单独记录。服务端排队/执行时间戳、实际 batching、固定到达速率和 -完整部署成本为 null。这些兼容新增字段保持 schema 版本 1.0.0,旧运行中不存在或为 null。 -比较时必须保持工作负载和计时边界一致;服务测量仅作描述,不能沿用串行回归校准。 - -积分复用 InferenceX 的梯形功率积分器,边界做线性插值,不做外推。UUID/进程 -归属、有限读数、时间戳顺序、阶段重叠、时钟一致性及最大间隔共同决定有效性。 -允许间隔为 `3 × 请求采样间隔`(本次运行为 3 秒)。旧版 UTC 事件重建必须与记录的单调 -时钟耗时一致;启动边界未被覆盖时,该阶段可能无效。H200 NVML 功率读数有向后 -平均的窗口,阶段边缘还受传感器平均窗口影响。不能从瓦数采样推断能量计数器。 - -## TDP 与架构结论 - -仅凭 `NVIDIA H200` 名称不能确定 SXM 形态或 700 W TDP。只有明确验证、附来源 -且绑定相同物理 UUID 的硬件记录,才能使 `hardware.tdp` 可用。此时导出器提供 -平均功率及观测峰值相对于所选 GPU 总规格 TDP 的比例。这些是描述性比例, -不是配置功率上限,也不是已校准的判定门槛。 - -后续只读硬件盘点保存在 `later_hardware_observation`,保留自身 CI/Slurm 身份和 -时间。其配置、默认、实际执行及最大功率上限**不能回填历史生成设置**。 -原始运行没有记录的历史上限仍为不可用。新运行会在 -`configured_power_limits.by_role` 中保留各角色执行前后的配置、实际执行、默认 -及最大功率上限(W)。每个快照具有观测时间和有效性;UUID、读数或时间不匹配 -时保留该快照为空。`same_observed_values` 只比较两个端点,不能证明中间功率上限 -始终不变。 - -观察到 H3 板级功率较高,只能说明该负载及硬件配置。声称其与 LLM 存在架构差异, -仍需匹配硬件/拓扑、精度、功率上限、采样与测量窗口、预热及负载,区分 LLM -prefill/decode,并进行重复可比测量。这些 A/A 点不证明统计显著性、通用视频 -质量、性能提升或发布就绪。 - -## 导出 API - -从仓库根目录执行,将仓库根目录及 `experimental/video-generation` 加入 -`PYTHONPATH`,然后调用: - -```python -from pathlib import Path -from evaluator.mvp_result import write_result - -write_result( - Path("/absolute/path/to/copied-source-artifact"), - producer={"git_commit": "<40-character exporter commit>", "ci": {"run_id": ""}}, - source_ci=trusted_github_run_metadata, - hardware_profile=optional_later_inventory, -) -``` - -使用新的副本。`source_ci` 使用 GitHub CLI 字段 `databaseId`、`runAttempt`、 -`headSha`、`url`、`status`、`conclusion`、`jobs`,必须独立于产物获取。 -可选硬件记录必须关联相同 GPU UUID;已验证的 TDP 需包含 `status`、 -`watts_per_gpu`、`hardware_variant`、`source_url` 和证据。 -发布方负责最终校验和、上传及下载验收。不得修改旧 CI 或测量身份,将导出伪装 -成新基准运行。 - -## 已观测的 A/A 证据 - -以下结果复用已保留的八秒机械狐狸运行: -[CI run 34293342829](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829), -提交 `65699f7c6`,Slurm **82261.0**。 -[原始媒体、遥测及报告](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34293342829/artifacts/10082823150) -可下载。四张选中的 H200 GPU 按顺序运行相同运行时版本,每个角色各执行一次预热 -和一次视频测量。 - -| 八秒视频测量 | Baseline | Candidate | -| --- | ---: | ---: | -| 从提交到媒体校验完成的延迟(s) | 149.768568 | 149.355090 | -| GPU 板级平均总功率(W) | 2737.371816 | 2731.735261 | -| 平均功率 / 已验证的总规格 TDP | 97.7633% | 97.5620% | -| 每个有效视频的 GPU 能量(J) | 406794.062298 | 404620.528709 | -| 采样覆盖比例 | 1.0 | 1.0 | -| 最大观测采样间隔(s) | 1.3303 | 1.3567 | - -每张 GPU 的平均功率约为 **680–688 W**。两个测量生成区间的起止边界均被遥测 -包围,最大采样间隔均低于 3 秒有效性上限。功率和能量覆盖提交到观测到提供方 -完成的区间;延迟还包含媒体传输及校验,两者测量边界不同。 - -后续[硬件盘点运行 34297499754](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754) -([原始盘点产物](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34297499754/artifacts/10083702100)) -在 `worker-10` 上完成 Slurm **82290.0**,于 `2026-09-09T01:02:44Z` 观测原来的 -四个 UUID。NVIDIA H200 PCI 设备/子系统编号 `233510DE` / `18BE10DE` 对应 SXM -板卡,厂家最大可配置 TDP 为[每张 700 W](https://www.nvidia.com/en-us/data-center/h200/), -四张所选 GPU 合计 **2800 W**。实测平均功率分别为此规格值的 -**97.7633% / 97.5620%**。按描述性的 90%“接近 TDP”标准,两者均满足,因此这些 -H3 生成测量区间支持 Oren 的假设。该标准不是经过校准的基准门槛,也不能证明 -与 LLM 存在架构差异。 - -后续盘点中,四张卡的配置、实际执行、默认及最大功率上限均为 700 W;原始生成 -时的设置仍未知。原始盘点 profile 因 PCI 编号未带 `0x` 而将形态记为 unknown; -导出器现用自身 producer 提交重新解析保留的 XML,保留源 profile,不发起新的 -GPU 查询。可使用[仅 CPU 的重放命令](https://github.com/SemiAnalysisAI/InferenceX/blob/a526907b1154901744bc6c058d213face64a3fef/experimental/video-generation/README_zh.md#结果与本地检查),将此次已验证 -盘点与原始 H3 产物一起复用。 - -此前四秒 A/A 运行保存在 -[CI run 34291306687](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687), -Slurm **82260.0**,并有 -[原始产物](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34291306687/artifacts/10081961245)。 -其 baseline 测量区间的结束边界未被遥测包围,因此功率及能量保留为空。 -这不会使已保留的负载执行、延迟和媒体结果失效。 - -每个时长都是单独固定的负载。不能合并这些时长的数据,也不能将两者延迟差异 -解释为回归。每个角色只有一个测量视频,只能提供点估计;负载执行通过,回归 -仍未校准,结论为 inconclusive。 - - -### 并发烟测矩阵 - -经审阅的 site 配置使用 `mode: serving-smoke` 时,在同一个分配中依次运行 -C1/C2/C4,每组恰好四个测量请求,预热单独记录。每组重启同一个 baseline -运行时;任何一组失败即停止。只申请配置指定的 GPU 数量。 -`serving-smoke.json` 汇总状态、吞吐和延迟,`gpu/cN/` 保留原始请求、视频和 -遥测,`report/index.html` 可播放原始媒体。中断请求与未启动请求分别计数。 -该模式不会导出配对回归结果,四个样本也不能认定 P90/P95 或持续服务容量。 diff --git a/experimental/video-generation/ci.py b/experimental/video-generation/ci.py index 60c9dc5f10..36dbd8f9c5 100644 --- a/experimental/video-generation/ci.py +++ b/experimental/video-generation/ci.py @@ -415,6 +415,9 @@ def prepared_spec(config: dict) -> dict: spec["gpu_uuids"] = [f"{prefix}00000000-0000-0000-0000-{i:012d}" for i in range(config["resources"]["gpus"])] from evaluator.mvp_gpu_job import validate_gpu_job spec = validate_gpu_job(spec) + from evaluator.mvp_runner import WAN_MODEL_ID + need(spec.get("plan", {}).get("model_id") != WAN_MODEL_ID or config["mode"] == "serving-smoke", + "Wan currently requires serving-smoke mode; paired quality qualification is unsupported") if config["mode"] == "serving-smoke": from evaluator.mvp_serving_smoke import validate_spec validate_spec(spec) @@ -716,6 +719,68 @@ def inside(run_dir: Path) -> int: return result["exit_code"] +def prepared_artifact_prefix(config: dict) -> str: + """Resolve the declared model without allocating or checking runtime capacity.""" + from evaluator.mvp_runner import WAN_MODEL_ID, validate_plan + + need(digest(config["spec"]["path"]) == config["spec"]["sha256"], "Prepared GPU specification changed") + plan = validate_plan(read(config["spec"]["path"])["plan"]) + if plan["model_id"] == WAN_MODEL_ID: + need(config["mode"] == "serving-smoke", "Wan currently requires serving-smoke mode") + return "video-serving" + return "h3-video" + + +def record_wan_preparation_failure(config: dict, output: Path, error: str) -> None: + """Retain declared Wan work after a pre-allocation failure, without execution claims.""" + if (output / "manifest.json").exists() or prepared_artifact_prefix(config) != "video-serving": + return + from evaluator.mvp_runner import validate_plan + from evaluator.mvp_serving_smoke import validate_concurrencies + + plan = validate_plan(read(config["spec"]["path"])["plan"]) + concurrencies = validate_concurrencies(config.get("concurrencies", (1, 2, 4))) + count = len(plan["cases"]) * plan["repetitions"] + need(4 <= count <= 200, "serving matrix requires 4–200 measured requests per configuration") + run_id, attempt, sha = (os.environ.get(key, "") for key in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "H3_SOURCE_SHA")) + need(run_id.isdigit() and attempt.isdigit() and re.fullmatch(r"[0-9a-f]{40}", sha), "CI identity required") + run_dir = Path(config["workspace"]["host"]) / "results" / config["task_id"] / f"github-{run_id}-{attempt}" + # A persistent run directory means execution may have started. Do not replace + # its evidence with a preparation-only failure receipt. + if run_dir.exists(): + return + repository = os.environ.get("GITHUB_REPOSITORY", "") + need(repository == "SemiAnalysisAI/InferenceX", "Expected InferenceX CI repository") + timestamp = now() + completion = {"scheduled": count, "attempted": 0, "completed": 0, "valid": 0, + "failed": count, "not_started": count, "unfinished": 0} + write(output / "serving-smoke.json", { + "schema_version": "1.0.0", "bundle_type": "video_serving_smoke_matrix", "status": "failed", + "plan": plan, "started_at": timestamp, "finished_at": timestamp, + "requests_per_configuration": count, "warmup_per_configuration": plan["warmup_runs"], + "scheduled": count * len(concurrencies), "ci_accepted": False, "release_qualified": False, + "error": error, "execution_started": False, + "cells": [{"concurrency": c, "status": "not_started", "verified": False, + "completion": dict(completion)} for c in concurrencies], + "completion": {key: value * len(concurrencies) for key, value in completion.items()}, + }) + write(output / "ci.json", { + "schema_version": 1, "task_id": config["task_id"], "run_id": run_id, "run_attempt": attempt, + "source_sha": sha, "mode": "serving-smoke", "phase": "failed", "site": config.get("site", DEFAULT_SITE), + "exit_code": 2, "error": error, "execution_started": False, "ci_accepted": False, + "release_qualified": False, + }) + write(output / "manifest.json", { + "schema_version": 1, "task_id": config["task_id"], "git_commit": sha, + "ci": {"repository": repository}, "run_id": run_id, "run_attempt": attempt, + "workload_plan": plan, "mode": "serving-smoke", "site": config.get("site", DEFAULT_SITE), + "exit_code": 2, "slurm_allocation": None, "execution_started": False, + "prepared_spec": config["spec"], "artifact_checksums": "SHA256SUMS", + "evidence": {name: digest(output / name) for name in ("ci.json", "serving-smoke.json")}, + }) + (output / "SHA256SUMS").write_text("".join(f"{value} {name}\n" for name, value in inventory(output).items())) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path) @@ -730,10 +795,18 @@ def main() -> int: return inside(args.inside) need(args.config is not None and args.output is not None, "--config and --output required") args.output.mkdir(parents=True, exist_ok=False) + config = None try: - return launch(read(args.config), args.output) + config = validate_config(read(args.config)) + return launch(config, args.output) except (Exception, KeyboardInterrupt) as error: write(args.output / "adapter-error.json", {"error": str(error), "exit_code": 2, "recorded_at": now(), "ci_accepted": False}) + if config is not None: + try: + record_wan_preparation_failure(config, args.output, str(error)) + except (OSError, ValueError, KeyError, TypeError): + # Invalid or unavailable declarations cannot establish a workload. + pass print(str(error), file=sys.stderr) return 2 diff --git a/experimental/video-generation/dashboard-observations.schema.json b/experimental/video-generation/dashboard-observations.schema.json new file mode 100644 index 0000000000..dd48ffc689 --- /dev/null +++ b/experimental/video-generation/dashboard-observations.schema.json @@ -0,0 +1,383 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SemiAnalysisAI/InferenceX/blob/main/experimental/video-generation/dashboard-observations.schema.json", + "title": "VideoGenX additive dashboard observations v1", + "description": "Optional sealed source-bound projection. Shape validation alone does not prove source hashes, hardware health or calibrated quality.", + "type": "object", + "additionalProperties": false, + "properties": { + "schemaVersion": { + "const": 1 + }, + "sourceRunId": { + "type": "string", + "pattern": "^[0-9]+$" + }, + "sourceSha": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "cells": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/$defs/cell" + } + } + }, + "required": [ + "schemaVersion", + "sourceRunId", + "sourceSha", + "cells" + ], + "$defs": { + "text": { + "type": [ + "string", + "null" + ], + "pattern": "\\S" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "nullableSha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "positiveInteger": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 9007199254740991 + }, + "cell": { + "type": "object", + "additionalProperties": false, + "properties": { + "runSha256": { + "$ref": "#/$defs/sha256" + }, + "specSha256": { + "$ref": "#/$defs/sha256" + }, + "deployment": { + "anyOf": [ + { + "$ref": "#/$defs/deployment" + }, + { + "type": "null" + } + ] + }, + "hardwareHealth": { + "anyOf": [ + { + "$ref": "#/$defs/hardwareHealth" + }, + { + "type": "null" + } + ] + }, + "quality": { + "anyOf": [ + { + "$ref": "#/$defs/quality" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "runSha256", + "specSha256" + ] + }, + "deployment": { + "type": "object", + "additionalProperties": false, + "properties": { + "gpusPerReplica": { + "$ref": "#/$defs/positiveInteger" + }, + "ring": { + "$ref": "#/$defs/positiveInteger" + }, + "cfg": { + "$ref": "#/$defs/positiveInteger" + }, + "batchSize": { + "$ref": "#/$defs/positiveInteger" + }, + "maxBatchSize": { + "$ref": "#/$defs/positiveInteger" + }, + "batchDelayMs": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "offload": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "ditCpu": { + "type": [ + "boolean", + "null" + ] + }, + "ditLayerwise": { + "type": [ + "boolean", + "null" + ] + }, + "textEncoderCpu": { + "type": [ + "boolean", + "null" + ] + }, + "imageEncoderCpu": { + "type": [ + "boolean", + "null" + ] + }, + "vaeCpu": { + "type": [ + "boolean", + "null" + ] + } + } + }, + { + "type": "null" + } + ] + }, + "encoderParallel": { + "$ref": "#/$defs/text" + }, + "scheduling": { + "$ref": "#/$defs/text" + }, + "engine": { + "$ref": "#/$defs/text" + }, + "precision": { + "$ref": "#/$defs/text" + }, + "acceleration": { + "$ref": "#/$defs/text" + }, + "generationKey": { + "$ref": "#/$defs/text" + }, + "configEvidence": { + "$ref": "#/$defs/text" + } + } + }, + "hardwareHealth": { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "pass", + "fail", + "unknown" + ] + }, + "reason": { + "$ref": "#/$defs/text" + }, + "evidence": { + "$ref": "#/$defs/text" + } + }, + "required": [ + "status" + ] + }, + "quality": { + "type": "object", + "additionalProperties": false, + "properties": { + "contractId": { + "$ref": "#/$defs/text" + }, + "contractSha256": { + "$ref": "#/$defs/nullableSha256" + }, + "rubricVersion": { + "$ref": "#/$defs/text" + }, + "rubricSha256": { + "$ref": "#/$defs/nullableSha256" + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "properties": { + "prompt_adherence": { + "$ref": "#/$defs/qualityMetric" + }, + "visual_fidelity": { + "$ref": "#/$defs/qualityMetric" + }, + "temporal_consistency": { + "$ref": "#/$defs/qualityMetric" + }, + "motion_plausibility": { + "$ref": "#/$defs/qualityMetric" + }, + "audio_quality": { + "$ref": "#/$defs/qualityMetric" + }, + "audio_content": { + "$ref": "#/$defs/qualityMetric" + }, + "av_sync": { + "$ref": "#/$defs/qualityMetric" + } + } + }, + "scale": { + "const": "ordinal_0_to_4" + } + }, + "required": [ + "scale", + "metrics" + ] + }, + "qualityMetric": { + "type": "object", + "additionalProperties": false, + "properties": { + "value": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 4 + }, + "status": { + "enum": [ + "unjudged", + "judged_unqualified", + "pass", + "fail", + "inconclusive", + "unassessable" + ] + }, + "direction": { + "enum": [ + "higher", + "lower" + ] + }, + "evaluatorId": { + "$ref": "#/$defs/text" + }, + "evaluatorVersion": { + "$ref": "#/$defs/text" + }, + "evaluatorSha256": { + "$ref": "#/$defs/nullableSha256" + }, + "samples": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 9007199254740991 + }, + "total": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 9007199254740991 + }, + "calibration": { + "anyOf": [ + { + "$ref": "#/$defs/calibration" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "status", + "direction" + ] + }, + "calibration": { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "enum": [ + "uncalibrated", + "calibrated" + ] + }, + "cohortId": { + "$ref": "#/$defs/text" + }, + "threshold": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 4 + }, + "provenance": { + "$ref": "#/$defs/text" + }, + "frozenAt": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + }, + "required": [ + "status" + ] + } + } +} diff --git a/experimental/video-generation/evaluator/cli.py b/experimental/video-generation/evaluator/cli.py index ba619565ee..da511b7901 100644 --- a/experimental/video-generation/evaluator/cli.py +++ b/experimental/video-generation/evaluator/cli.py @@ -81,6 +81,17 @@ def _mvp_command(arguments: argparse.Namespace) -> int: return 1 return 2 + if arguments.command == "serving-smoke": + from .mvp_serving_smoke import run_matrix, validate_concurrencies, validate_spec + + spec = validate_spec(_load_mvp_object(arguments.spec)) + concurrencies = validate_concurrencies(arguments.concurrencies) + arguments.output.mkdir(parents=True, exist_ok=False) + result = run_matrix(spec, arguments.output, concurrencies=concurrencies) + print(json.dumps({"status": result["status"], "completion": result["completion"]}, + indent=2, sort_keys=True, allow_nan=False)) + return 0 if result["status"] == "complete" else 1 + if arguments.command == "srt-gpu-job": from .mvp_srt import run_srt_gpu_job @@ -256,6 +267,11 @@ def _parser() -> argparse.ArgumentParser: gpu_job.add_argument("--execute", action="store_true") gpu_job.add_argument("--output", type=Path, help="new evidence directory; no overwrites") + serving_smoke = subparsers.add_parser("serving-smoke", help="run a bounded H3 or Wan serving matrix on allocated GPUs") + serving_smoke.add_argument("--spec", required=True, type=Path) + serving_smoke.add_argument("--output", required=True, type=Path, help="new evidence directory; no overwrites") + serving_smoke.add_argument("--concurrencies", nargs="+", type=int, default=[1, 2, 4]) + gpu_report = subparsers.add_parser("gpu-report", help="render recorded GPU job evidence, including incomplete runs") gpu_report.add_argument("job", type=Path) gpu_report.add_argument("--output", required=True, type=Path, help="new report directory") @@ -325,10 +341,10 @@ def _parser() -> argparse.ArgumentParser: srt_finalize.add_argument("--concurrencies", nargs="+", type=int, required=True) srt_finalize.add_argument("--output", required=True, type=Path) - run = subparsers.add_parser("run", help="preview an H3 plan; --execute explicitly submits it") + run = subparsers.add_parser("run", help="preview an H3 or Wan plan; --execute explicitly submits it") run.add_argument("plan", type=Path) run.add_argument("--runtime", choices=("sglang", "vllm-omni"), default="sglang") - run.add_argument("--execute", action="store_true", help="submit real requests to your H3 server") + run.add_argument("--execute", action="store_true", help="submit real requests to your H3 or Wan server") run.add_argument("--endpoint", help="self-hosted server base URL, without /v1/videos") run.add_argument("--runtime-revision", help="operator-declared exact server code/container revision") run.add_argument("--hardware-label", help="operator-declared hardware, precision, and topology label") diff --git a/experimental/video-generation/evaluator/mvp_deployment_frontier.py b/experimental/video-generation/evaluator/mvp_deployment_frontier.py new file mode 100644 index 0000000000..2591187941 --- /dev/null +++ b/experimental/video-generation/evaluator/mvp_deployment_frontier.py @@ -0,0 +1,295 @@ +"""Offline deployment frontiers over normalized, provenance-checked observations. + +This reader selects existing measurements; it never launches a workload, infers +missing metrics, or qualifies quality. The producer supplies canonical workload / +generation and deployment identities, queueing status, and (when required) raw +seven-dimension assessment records from its versioned quality protocol. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from datetime import datetime +from pathlib import Path +from typing import Any, Literal + +Direction = Literal["lower", "higher"] +Point = dict[str, Any] +QUALITY_METRIC_IDS = ( + "prompt_adherence", + "visual_fidelity", + "temporal_consistency", + "motion_plausibility", + "audio_quality", + "audio_content", + "av_sync", +) +QUALITY_SCALE = "ordinal_0_to_4" +QUALITY_REGISTRY_METRIC = "human.absolute_dimension_rating" +QUALITY_REGISTRY_VERSION = "0.2.0-draft" + + +def _finite(value: Any) -> bool: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _hash(value: Any) -> bool: + return isinstance(value, str) and re.fullmatch(r"[a-f0-9]{64}", value) is not None + + +def _score(value: Any) -> bool: + return _finite(value) and 0 <= value <= 4 + + +def _date(value: Any) -> bool: + if not _text(value): + return False + try: + # Staged helpers may run in the retained Python 3.10 AMD environment. + datetime.fromisoformat(value.replace("Z", "+00:00")) # noqa: FURB162 + except ValueError: + return False + return True + + +def qualified_quality_cohort( + point: Point, metric: str = "prompt_adherence", threshold: float | None = None +) -> str | None: + """Match the App's seven-dimension admission and complete frozen-rule identity. + + This checks recorded evidence fields, not whether the producer's judgments or + calibration are scientifically valid. The reader threshold only tightens the + selected dimension. An opaque producer qualification string is not evidence. + """ + if metric not in QUALITY_METRIC_IDS: + raise ValueError("Unknown quality dimension") + if threshold is not None and not _score(threshold): + return None + assessment = point.get("quality") + if not isinstance(assessment, dict) or assessment.get("scale") != QUALITY_SCALE: + return None + if not ( + _text(assessment.get("contractId")) + and _hash(assessment.get("contractSha256")) + and _text(assessment.get("rubricVersion")) + and _hash(assessment.get("rubricSha256")) + and isinstance(assessment.get("metrics"), dict) + ): + return None + dimensions = [] + for dimension in QUALITY_METRIC_IDS: + result = assessment["metrics"].get(dimension) + if not isinstance(result, dict): + return None + samples = result.get("samples") + calibration = result.get("calibration") + if not ( + result.get("status") == "pass" + and result.get("direction") == "higher" + and _score(result.get("value")) + and _text(result.get("evaluatorId")) + and _text(result.get("evaluatorVersion")) + and _hash(result.get("evaluatorSha256")) + and _finite(samples) + and 0 < samples <= 9007199254740991 + and int(samples) == samples + and _finite(result.get("total")) + and samples == result["total"] + and ( + "samples" not in point + or (_finite(point["samples"]) and result["total"] == point["samples"]) + ) + and isinstance(calibration, dict) + and calibration.get("status") == "calibrated" + and _text(calibration.get("cohortId")) + and _score(calibration.get("threshold")) + and _text(calibration.get("provenance")) + and _date(calibration.get("frozenAt")) + ): + return None + frozen_threshold = calibration["threshold"] + if result["value"] < frozen_threshold or ( + dimension == metric + and threshold is not None + and result["value"] < threshold + ): + return None + # JSON.stringify renders integral thresholds without a decimal suffix. + if int(frozen_threshold) == frozen_threshold: + frozen_threshold = int(frozen_threshold) + dimensions.append( + json.dumps( + [ + assessment["contractId"], + assessment["contractSha256"], + assessment["rubricVersion"], + assessment["rubricSha256"], + dimension, + QUALITY_REGISTRY_METRIC, + QUALITY_REGISTRY_VERSION, + result["direction"], + result["evaluatorId"], + result["evaluatorVersion"], + result["evaluatorSha256"], + calibration["cohortId"], + frozen_threshold, + calibration["provenance"], + calibration["frozenAt"], + ], + separators=(",", ":"), + ensure_ascii=False, + ) + ) + return json.dumps( + [QUALITY_SCALE, metric, *dimensions], separators=(",", ":"), ensure_ascii=False + ) + + +def pareto_frontier( + points: list[Point], x_better: Direction, y_better: Direction +) -> list[Point]: + """Non-dominated finite measurements, preserving exact ties and sorting by x.""" + if x_better not in ("lower", "higher") or y_better not in ("lower", "higher"): + raise ValueError("Axis directions must be lower or higher") + measured = [ + point for point in points if _finite(point.get("x")) and _finite(point.get("y")) + ] + sx = 1 if x_better == "higher" else -1 + sy = 1 if y_better == "higher" else -1 + # Deployment matrices are small; explicit strict dominance preserves ties + # without treating a different measured configuration as a duplicate. + frontier = [ + point + for point in measured + if not any( + sx * other["x"] >= sx * point["x"] + and sy * other["y"] >= sy * point["y"] + and (other["x"] != point["x"] or other["y"] != point["y"]) + for other in measured + ) + ] + return sorted(frontier, key=lambda point: point["x"]) + + +def select_frontiers( + points: list[Point], + x_better: Direction, + y_better: Direction, + *, + quality_required: bool = False, + quality_metric: str = "prompt_adherence", + quality_threshold: float | None = None, + optimal: bool = True, +) -> dict[str, Any]: + """Apply admission before dominance and keep hardware/cohort lines separate. + + Quality admission derives the cohort from all seven recorded dimensions after + threshold, coverage and calibration checks. Missing evidence fails closed; + this reader does not establish the truth of judgments or calibration. + """ + eligible: list[Point] = [] + excluded: dict[str, str] = {} + by_hardware: dict[str, dict[str, list[Point]]] = {} + for source in points: + for name in ("id", "hardwareKey", "deploymentKey"): + if not isinstance(source.get(name), str) or not source[name]: + raise ValueError(f"Each observation needs a nonempty {name}") + health = source.get("hardwareHealth") or {} + queueing = source.get("queueingStatus") + quality = ( + qualified_quality_cohort(source, quality_metric, quality_threshold) + if quality_required + else None + ) + reason = None + if health.get("status") == "fail": + reason = "hardware_health_failed" + elif queueing != "unqueued": + reason = "queueing" if queueing == "queueing" else "unknown_capacity" + elif quality_required and (not isinstance(quality, str) or not quality): + reason = "quality_unqualified" + elif not _finite(source.get("x")) or not _finite(source.get("y")): + reason = "missing_axis" + if reason: + excluded[source["id"]] = reason + continue + point = {**source, "optimal": False} + if quality_required: + point["qualifiedQualityCohort"] = quality + eligible.append(point) + identity = point.get("cohortKey") + cohort = ( + identity + if isinstance(identity, str) and identity + else f"unknown:{point['id']}" + ) + if quality_required: + cohort = json.dumps([cohort, quality]) + by_hardware.setdefault(point["hardwareKey"], {}).setdefault(cohort, []).append( + point + ) + frontiers: dict[str, list[Point]] = {} + multi_layout = False + for hardware, cohorts in by_hardware.items(): + for index, cohort in enumerate(cohorts.values()): + key = hardware if len(cohorts) == 1 else f"{hardware}:{index}" + frontier = pareto_frontier(cohort, x_better, y_better) + for point in frontier: + point["optimal"] = True + if len({point["deploymentKey"] for point in frontier}) < len(frontier): + for measurement, point in enumerate(frontier): + frontiers[f"{key}:observation{measurement}"] = [point] + else: + frontiers[key] = frontier + multi_layout |= len({point["deploymentKey"] for point in cohort}) > 1 + return { + "plotted": [point for point in eligible if not optimal or point["optimal"]], + "frontiers": frontiers, + "multiLayout": multi_layout, + "excluded": excluded, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "source", type=Path, help="JSON object containing normalized points" + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--x-better", choices=("lower", "higher"), required=True) + parser.add_argument("--y-better", choices=("lower", "higher"), required=True) + parser.add_argument("--quality-required", action="store_true") + parser.add_argument( + "--quality-metric", choices=QUALITY_METRIC_IDS, default="prompt_adherence" + ) + parser.add_argument("--quality-threshold", type=float) + parser.add_argument("--include-dominated", action="store_true") + args = parser.parse_args() + if args.source.resolve() == args.output.resolve(): + parser.error("Output must differ from the retained input") + records = json.loads(args.source.read_text()) + result = select_frontiers( + records["points"], + args.x_better, + args.y_better, + quality_required=args.quality_required, + quality_metric=args.quality_metric, + quality_threshold=args.quality_threshold, + optimal=not args.include_dominated, + ) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") + + +if __name__ == "__main__": + main() diff --git a/experimental/video-generation/evaluator/mvp_gpu_evidence.py b/experimental/video-generation/evaluator/mvp_gpu_evidence.py index fcbe83d1d3..c5a046e6af 100644 --- a/experimental/video-generation/evaluator/mvp_gpu_evidence.py +++ b/experimental/video-generation/evaluator/mvp_gpu_evidence.py @@ -14,7 +14,7 @@ from datetime import datetime from pathlib import Path -from .mvp_runner import _slots, canonical_json_bytes +from .mvp_runner import WAN_MODEL_ID, _slots, canonical_json_bytes def _finite(value, *, positive=False): @@ -61,6 +61,9 @@ def verify_measurement_job(directory: Path, *, deadline: float, require_success: directory = Path(directory).resolve(strict=True) gpu._check_deadline(deadline) spec = gpu.validate_gpu_job(gpu._read(_file(directory, "spec.json"))) + wan = spec["plan"]["model_id"] == WAN_MODEL_ID + if wan and not serving_smoke: + raise ValueError("Wan is supported only in controlled serving smoke; paired qualification is unavailable") receipt = gpu._read(_file(directory, "gpu-job.json")) bundle_type = "controlled_serving_smoke" if serving_smoke else "controlled_gpu_job" if serving_smoke and not spec.get("serving"): @@ -69,7 +72,8 @@ def verify_measurement_job(directory: Path, *, deadline: float, require_success: raise ValueError("GPU job/spec identity is not verified") if receipt.get("job_id") != spec["job_id"] or receipt.get("plan_sha256") != gpu._digest(spec["plan"]): raise ValueError("GPU job workload identity mismatch") - if receipt.get("evidence_kind") != "controlled_h3_gpu" or receipt.get("status") != "complete" or receipt.get("measurement_status") != "complete" or receipt.get("failures"): + controlled_kind = "controlled_video_gpu" if wan else "controlled_h3_gpu" + if receipt.get("evidence_kind") != controlled_kind or receipt.get("status") != "complete" or receipt.get("measurement_status") != "complete" or receipt.get("failures"): raise ValueError("GPU job has incomplete, fixture, or failed supervision evidence") if receipt.get("cleanup_status") != "clean": raise ValueError("GPU job did not complete clean owned-resource teardown") @@ -114,7 +118,8 @@ def verify_measurement_job(directory: Path, *, deadline: float, require_success: if gpu._hash(run_path, deadline) != role.get("run_sha256"): raise ValueError("role run bundle hash mismatch") run = gpu._read(run_path) - if run.get("bundle_type") != "mvp_run" or run.get("bundle_version") != "0.1.0" or run.get("evidence_kind") not in {"live_h3", "operator_endpoint"}: + live_kind = "live_video" if wan else "live_h3" + if run.get("bundle_type") != "mvp_run" or run.get("bundle_version") != "0.1.0" or run.get("evidence_kind") not in {live_kind, "operator_endpoint"}: raise ValueError("role contains fixture/imported or unsupported run evidence") if run.get("status") not in {"complete", "partial", "failed"} or _date(run.get("finished_at")) < _date(run.get("started_at")): raise ValueError("role run is not finalized") diff --git a/experimental/video-generation/evaluator/mvp_gpu_job.py b/experimental/video-generation/evaluator/mvp_gpu_job.py index 16e583434b..1813f4ea19 100644 --- a/experimental/video-generation/evaluator/mvp_gpu_job.py +++ b/experimental/video-generation/evaluator/mvp_gpu_job.py @@ -1,4 +1,4 @@ -"""Bounded, locally supervised H3 GPU measurements on a trusted Linux runner. +"""Bounded, locally supervised video GPU measurements on a trusted Linux runner. This is NOT a sandbox for untrusted runtime code. The operator must provision the source trees, environments, weights, and (for CI acceptance) a dedicated @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any -from .mvp_runner import canonical_json_bytes, preview_plan, validate_plan +from .mvp_runner import WAN_MODEL_ID, canonical_json_bytes, preview_plan, validate_plan VERSION = "0.1.0" @@ -223,8 +223,21 @@ def validate_gpu_job(spec: dict) -> dict: raise ValueError("model manifest contains no weight files") frozen["model"]["files"] = sorted(files, key=lambda item: item["path"]) server = frozen["server"] - _keys(server, {"ulysses_degree", "tp_size", "encoder_parallel", "performance_mode"}, "server", - optional={"dit_cpu_offload", "layerwise_offload", "attention_backend"}) + wan = frozen["plan"]["model_id"] == WAN_MODEL_ID + if wan: + _keys(server, {"ulysses_degree", "tp_size", "dit_cpu_offload", "dit_layerwise_offload", "text_encoder_cpu_offload", "vae_cpu_offload"}, "Wan server") + if not frozen.get("serving") or vendor != "nvidia" or frozen.get("server_timing"): + raise ValueError("Wan requires NVIDIA serving smoke without the H3 server timing hook") + for field in ("dit_cpu_offload", "text_encoder_cpu_offload", "vae_cpu_offload"): + if type(server[field]) is not bool: + raise ValueError(f"Wan server.{field} must be an explicit boolean") + if server["dit_layerwise_offload"] is not False: + raise ValueError("Wan server.dit_layerwise_offload must be explicitly false in this smoke path") + if len(devices) not in {1, 2, 4, 8} or server["tp_size"] != 1 or server["ulysses_degree"] != len(devices): + raise ValueError("Wan currently requires 1/2/4/8 GPUs with TP1 and one Ulysses group") + else: + _keys(server, {"ulysses_degree", "tp_size", "encoder_parallel", "performance_mode"}, "server", + optional={"dit_cpu_offload", "layerwise_offload", "attention_backend"}) if vendor == "amd": if server.get("attention_backend") != "aiter" or server["tp_size"] != 1 or server["ulysses_degree"] != len(devices): raise ValueError("AMD H3 currently requires the documented AITER / pure Ulysses layout") @@ -234,7 +247,7 @@ def validate_gpu_job(spec: dict) -> dict: _number(server["tp_size"], "server.tp_size", 1, len(devices), True) if len(devices) % server["ulysses_degree"] or len(devices) % server["tp_size"]: raise ValueError("server requires dividing Ulysses and tensor-parallel degrees") - if server["encoder_parallel"] not in {"auto", "fold", "replicate"} or server["performance_mode"] not in {"manual", "speed", "memory"}: + if not wan and (server["encoder_parallel"] not in {"auto", "fold", "replicate"} or server["performance_mode"] not in {"manual", "speed", "memory"}): raise ValueError("unsupported explicit encoder parallelism or performance mode") if "layerwise_offload" in server: # This is one documented lossless placement, not a general argument @@ -267,8 +280,15 @@ def validate_gpu_job(spec: dict) -> dict: raise ValueError("telemetry plan exceeds 100000 samples") if frozen["plan"]["warmup_runs"] < 1: raise ValueError("controlled GPU measurements require a separately recorded warmup") - from .mvp_compare import _policy - frozen["policy"] = _policy(frozen["policy"]) + if wan: + policy = frozen["policy"] + _keys(policy, {"policy_id", "calibration_status"}, "Wan policy") + if (not isinstance(policy["policy_id"], str) or not policy["policy_id"].strip() + or len(policy["policy_id"]) > 2000 or policy["calibration_status"] != "uncalibrated"): + raise ValueError("Wan policy requires a bounded identifier and uncalibrated status; no quality qualification is supported") + else: + from .mvp_compare import _policy + frozen["policy"] = _policy(frozen["policy"]) if frozen.get("serving") and frozen["policy"]["calibration_status"] == "operator_calibrated": raise ValueError("serving load requires an uncalibrated policy; serial calibration cannot qualify concurrent delivery metrics") memory_gate = frozen["policy"].get("max_memory_increase_fraction") @@ -277,7 +297,22 @@ def validate_gpu_job(spec: dict) -> dict: return frozen -def _server_argv(spec: dict, role: str, *, host: str = "127.0.0.1") -> list[str]: +def _server_argv(spec: dict, role: str, *, host: str = "127.0.0.1", output_path: str | None = None) -> list[str]: + if spec["plan"]["model_id"] == WAN_MODEL_ID: + if not output_path: + raise ValueError("Wan requires a job-owned server output path") + server = spec["server"] + return [spec[role]["python"], "-c", _LAUNCH, "serve", "--model-type", "diffusion", + "--model-path", spec["model"]["path"], "--model-id", spec["plan"]["model_id"], + "--revision", spec["model"]["revision"], "--num-gpus", str(len(spec["gpu_uuids"])), + "--ulysses-degree", str(server["ulysses_degree"]), "--tp-size", str(server["tp_size"]), + "--ring-degree", "1", "--enable-cfg-parallel", "false", + "--host", host, "--port", str(spec["port"]), "--enable-torch-compile", "false", + "--dit-cpu-offload", str(server["dit_cpu_offload"]).lower(), + "--dit-layerwise-offload", "false", + "--text-encoder-cpu-offload", str(server["text_encoder_cpu_offload"]).lower(), + "--vae-cpu-offload", str(server["vae_cpu_offload"]).lower(), + "--warmup-mode", "off", "--output-path", output_path] args = [spec[role]["python"], "-c", _LAUNCH, "serve", "--model-type", "diffusion", "--model-path", spec["model"]["path"], "--model-id", spec["plan"]["model_id"], "--revision", spec["model"]["revision"], "--model-variant", "fl2va", @@ -299,15 +334,17 @@ def _server_argv(spec: dict, role: str, *, host: str = "127.0.0.1") -> list[str] def preview_gpu_job(spec: dict) -> dict: frozen = validate_gpu_job(spec) + wan = frozen["plan"]["model_id"] == WAN_MODEL_ID + roles = ("baseline",) if wan else _ROLES return { "evidence_kind": "gpu_job_preview_no_execution", "spec_sha256": _digest(frozen), "job_id": frozen["job_id"], "gpu_uuids": frozen["gpu_uuids"], "authorization": frozen["authorization"], "authorization_verification": "operator assertion; not proof of model-license rights", "allocation": frozen["allocation"], "allocation_verification": "operator-declared prerequisite, not scheduler attestation", - "commands": {role: _server_argv(frozen, role) for role in _ROLES}, + "commands": {role: _server_argv(frozen, role, output_path=f"/supervisor/{role}/generated") for role in roles}, "workload": preview_plan(frozen["plan"]), "limits": frozen["limits"], - "sequence": ["verify pinned files", "acquire UUID locks", "verify idle", "baseline startup/warmup/measure/cleanup", "candidate startup/warmup/measure/cleanup", "compare"], - "warnings": ["Trusted Linux runner and visible GPU-process PIDs are required.", "No GPU lock can enforce exclusive access against non-cooperating processes.", "No GPU work, model download, server startup, or CI acceptance occurs in preview."], + "sequence": ["verify pinned files", "acquire UUID locks", "verify idle", "baseline startup/warmup/measure/cleanup"] + ([] if wan else ["candidate startup/warmup/measure/cleanup", "compare"]), + "warnings": ["Trusted Linux runner and visible GPU-process PIDs are required.", "No GPU lock can enforce exclusive access against non-cooperating processes.", "No GPU work, model download, server startup, or CI acceptance occurs in preview."] + ([" is resolved under the new job directory at execution; Wan supports serving smoke only."] if wan else []), } @@ -1120,9 +1157,10 @@ def _role(spec: dict, label: str, directory: Path, supervisor: _Supervisor, prob boot_started = time.monotonic() role["startup_timing_window"] = {"start_monotonic_seconds": boot_started, "start_utc": _now(), "end_monotonic_seconds": None} - owner = supervisor.spawn(_server_argv(spec, label), cwd=metadata, env=env, + launch_argv = _server_argv(spec, label, output_path=str((metadata / "generated").resolve())) + owner = supervisor.spawn(launch_argv, cwd=metadata, env=env, stdout=metadata / "runtime.stdout.log", stderr=metadata / "runtime.stderr.log", nonce=nonce) - role.update(status="starting", process_identity=owner.identity, launch_argv=_server_argv(spec, label), started_at=_now()) + role.update(status="starting", process_identity=owner.identity, launch_argv=launch_argv, started_at=_now()) _write(directory / "gpu-job.json", receipt) sampler = _Sampler(probe, owner, metadata / "telemetry.jsonl", spec["limits"]["telemetry_interval_seconds"], initial_sample=snapshot) @@ -1171,7 +1209,8 @@ def _role(spec: dict, label: str, directory: Path, supervisor: _Supervisor, prob run = _read(run_path) role["run_sha256"] = _hash(run_path) role["run_summary"] = run.get("summary") - if run.get("evidence_kind") not in {"operator_endpoint", "live_h3"} or run.get("plan_sha256") != _digest(spec["plan"]): + live_kind = "live_video" if spec["plan"]["model_id"] == WAN_MODEL_ID else "live_h3" + if run.get("evidence_kind") not in {"operator_endpoint", live_kind} or run.get("plan_sha256") != _digest(spec["plan"]): raise RuntimeError("supervised client returned non-endpoint or wrong-plan evidence") role["run_status"] = run.get("status") finalized = bool(run.get("finished_at") and run.get("status") in {"complete", "partial", "failed"}) @@ -1307,7 +1346,7 @@ def _require_linux(): def run_gpu_job(spec: dict, output_dir: Path, *, serving_smoke: bool = False) -> dict: - """Launch owned H3 sessions, or one serving smoke session; Linux-only. + """Launch owned H3 sessions, or one H3/Wan serving smoke session; Linux-only. Calling this function is execution authorization. CLI callers must put an explicit --execute barrier in front of it. No provisioning/download occurs. @@ -1320,6 +1359,9 @@ def run_gpu_job(spec: dict, output_dir: Path, *, serving_smoke: bool = False) -> acceptance after compute was consumed. """ spec = validate_gpu_job(spec) + wan = spec["plan"]["model_id"] == WAN_MODEL_ID + if wan and not serving_smoke: + raise ValueError("Wan is supported only in controlled serving smoke; paired qualification is unavailable") if serving_smoke and not spec.get("serving"): raise ValueError("single-runtime smoke requires an explicit serving load") approval = spec["authorization"] @@ -1372,12 +1414,12 @@ def run_gpu_job(spec: dict, output_dir: Path, *, serving_smoke: bool = False) -> for role in (("baseline",) if serving_smoke else _ROLES): supervisor.check() _role(spec, role, directory, supervisor, probe, receipt) - receipt["evidence_kind"] = "controlled_h3_gpu" + receipt["evidence_kind"] = "controlled_video_gpu" if wan else "controlled_h3_gpu" _write(directory / "gpu-job.json", receipt) # Model mutation during measurement invalidates the pinned identity. if _model_manifest(spec, supervisor.deadline, supervisor.cancelled) != receipt["model_identity"]: raise RuntimeError("staged model identity changed during measurement") - receipt.update(measurement_status="complete", evidence_kind="controlled_h3_gpu", cleanup_status="clean") + receipt.update(measurement_status="complete", evidence_kind="controlled_video_gpu" if wan else "controlled_h3_gpu", cleanup_status="clean") _write(directory / "gpu-job.json", receipt) finally: if any(role.get("cleanup", {}).get("status") == "failed" for role in receipt["roles"].values()): diff --git a/experimental/video-generation/evaluator/mvp_media.py b/experimental/video-generation/evaluator/mvp_media.py index b9e778a4c9..23278e6147 100644 --- a/experimental/video-generation/evaluator/mvp_media.py +++ b/experimental/video-generation/evaluator/mvp_media.py @@ -74,9 +74,11 @@ def _validate_expected(expected: dict) -> None: raise ValueError(f"expected.{key} must be an integer") if key == "max_frozen_fraction" and value > 1: raise ValueError("expected.max_frozen_fraction must be <=1") - for key in {"audio_required", "requires_motion", "requires_sound"} & expected.keys(): + for key in {"audio_required", "audio_forbidden", "requires_motion", "requires_sound"} & expected.keys(): if not isinstance(expected[key], bool): raise ValueError(f"expected.{key} must be a boolean") + if expected.get("audio_forbidden") and (expected.get("audio_required") or expected.get("requires_sound")): + raise ValueError("audio cannot be both forbidden and required") def _audio_array(frame: Any, np: Any) -> Any: @@ -403,6 +405,8 @@ def analyze_media(path: Path, expected: dict | None = None) -> dict: if "audio_required" in expected: required = bool(expected["audio_required"]) checks.append(_check("audio.required", bool(audio.get("present")) if required else None, audio.get("present", False), required, "False means audio is optional, not prohibited.")) + if expected.get("audio_forbidden"): + checks.append(_check("audio.absent", not audio.get("present"), audio.get("present", False), False, "The model contract is video-only; audio is unsupported.")) if audio.get("present"): checks.append(_check("audio.samples", audio.get("sample_count", 0) > 0, audio.get("sample_count", 0), ">0", "An advertised audio stream must contain decoded samples.")) timestamp_ok = audio["missing_timestamps"] == 0 and audio["nonincreasing_timestamps"] == 0 diff --git a/experimental/video-generation/evaluator/mvp_runner.py b/experimental/video-generation/evaluator/mvp_runner.py index 11d09f0dfb..4c73a32685 100644 --- a/experimental/video-generation/evaluator/mvp_runner.py +++ b/experimental/video-generation/evaluator/mvp_runner.py @@ -1,14 +1,14 @@ -"""Auditable serial and closed-loop serving client for an operator-managed H3 video endpoint. +"""Auditable serial and closed-loop client for an operator-managed H3 or Wan endpoint. This module does not launch a server, download weights, verify server identity, or contact MiniMax's paid API. ``preview_plan`` is network-free; ``run_plan`` submits real requests to the explicitly supplied endpoint. Local HTTP fixtures can exercise that transport, but are not evidence of H3 inference. -Protocol implementations were inspected at SGLang 253020450290328e9deb307eece1e402fa17f35e -and vLLM-Omni eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c. They are deliberately -separate: SGLang accepts the H3 JSON contract, while vLLM-Omni expects multipart -form fields. Those source pins do not attest to a supplied server's revision. +``_TRANSPORTS`` lists the audited (model, runtime) pairs with the protocol +sources inspected for each; every other pair is rejected. SGLang accepts a JSON +body, vLLM-Omni expects multipart form fields. Those source pins do not attest +to a supplied server's revision. """ from __future__ import annotations @@ -36,6 +36,7 @@ MODEL_ID = "MiniMaxAI/MiniMax-H3" +WAN_MODEL_ID = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" RUNTIMES = {"sglang", "vllm-omni"} MAX_JSON_BYTES = 1024 * 1024 MAX_MEDIA_BYTES = 512 * 1024 * 1024 @@ -45,8 +46,8 @@ POLL_INTERVAL_SECONDS = 0.5 _CHUNK_BYTES = 64 * 1024 _EVENT_LOCK = threading.Lock() -_PROTOCOL_SOURCES = { - "sglang": { +_TRANSPORTS: dict[tuple[str, str], dict[str, Any]] = { + (MODEL_ID, "sglang"): { "repository": "https://github.com/sgl-project/sglang", "revision": "253020450290328e9deb307eece1e402fa17f35e", "files": [ @@ -54,7 +55,7 @@ "python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/video_adapter.py", ], }, - "vllm-omni": { + (MODEL_ID, "vllm-omni"): { "repository": "https://github.com/vllm-project/vllm-omni", "revision": "eb11446b7f2e30ca582f8aff3afe12e9a2e66f6c", "files": [ @@ -62,6 +63,25 @@ "vllm_omni/entrypoints/openai/serving_video.py", ], }, + (WAN_MODEL_ID, "sglang"): { + "repository": "https://github.com/sgl-project/sglang", + "revision": "71de97b264b04dcd514cf904003028aefe9775c8", + "files": ["python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py", + "python/sglang/multimodal_gen/configs/sample/wan.py"], + }, + # vLLM-Omni v0.30.0: POST /v1/videos multipart fields (_parse_video_form, helpers.py:852-895) feed + # VideoGenerationRequest (protocol/videos.py:153-242), which accepts every Wan payload key; FastAPI drops + # the empty negative_prompt field and pipeline_wan2_2.py encodes "" for it. api_server.py serves the + # GET /v1/videos/{id} poll and /content download. + (WAN_MODEL_ID, "vllm-omni"): { + "repository": "https://github.com/vllm-project/vllm-omni", + "revision": "a8576ccb725c4e21cd13c3eb5f9a546b21149d2b", + "files": ["vllm_omni/entrypoints/openai/api_server.py", + "vllm_omni/entrypoints/openai/protocol/videos.py", + "vllm_omni/entrypoints/openai/video/generation/helpers.py", + "vllm_omni/entrypoints/openai/serving_video.py", + "vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py"], + }, } @@ -92,25 +112,7 @@ def _text(value: Any, name: str, *, limit: int = 1000) -> None: raise ValueError(f"{name} must be a nonempty string of at most {limit} characters") -def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: - """Validate the supported frozen T2VA subset without making a request. - - This is the execution safety contract; the broader study/registry schemas - remain separate. Unknown root metadata is preserved in the plan digest. - """ - if not isinstance(plan, dict): - raise ValueError("plan must be an object") - try: - frozen = json.loads(canonical_json_bytes(plan)) - except (TypeError, ValueError, OverflowError) as exc: - raise ValueError("plan must contain finite JSON values") from exc - for field in ("plan_id", "model_id", "model_revision"): - _text(frozen.get(field), field) - if frozen["model_id"] != MODEL_ID: - raise ValueError(f"this narrow MVP supports only {MODEL_ID}") - if not re.fullmatch(r"[a-fA-F0-9]{40}", frozen["model_revision"]): - raise ValueError("model_revision must be an immutable 40-character commit") - generation = frozen.get("generation") +def _validate_h3_generation(generation: dict[str, Any]) -> None: if not isinstance(generation, dict): raise ValueError("generation must be an object with explicit controls") allowed_controls = { @@ -142,6 +144,62 @@ def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: if (any(generation[key] != value for key, value in audited_cell.items()) or audited_frames.get(generation["duration_seconds"]) != generation["frame_count"]): raise ValueError("this MVP requires a 16:9, 1344x768 H3 cell: 4s/107 frames or 8s/192 frames") + + +def _validate_wan_generation(generation: dict[str, Any]) -> None: + controls = {"width", "height", "frame_count", "fps", "num_inference_steps", + "guidance_scale", "guidance_scale_2", "flow_shift", "negative_prompt"} + if not isinstance(generation, dict) or set(generation) != controls: + raise ValueError("Wan requires explicit video-only generation controls; unsupported fields are rejected") + for field, maximum in (("width", 8192), ("height", 8192), ("frame_count", 10000), + ("fps", 240), ("num_inference_steps", 1000)): + _positive(generation[field], f"generation.{field}", maximum=maximum, integer=True) + if generation["width"] % 16 or generation["height"] % 16: + raise ValueError("Wan width and height must be multiples of 16 to avoid runtime resizing") + if generation["frame_count"] < 5 or (generation["frame_count"] - 1) % 4: + raise ValueError("Wan frame_count must be 4n+1 with n >= 1 to avoid runtime rounding") + for field in ("guidance_scale", "guidance_scale_2", "flow_shift"): + _positive(generation[field], f"generation.{field}", maximum=1000) + if generation["guidance_scale"] <= 1: + raise ValueError("Wan guidance_scale must exceed 1 so both expert guidance scales are applied") + if generation["flow_shift"] != 12: + raise ValueError("The audited Wan runtime uses server flow_shift 12; per-request overrides are unsupported") + if not isinstance(generation["negative_prompt"], str) or len(generation["negative_prompt"]) > 32000: + raise ValueError("generation.negative_prompt must be an explicit string of at most 32000 characters") + + +def _transport(plan: dict[str, Any], runtime: str) -> dict[str, Any]: + if runtime not in RUNTIMES: + raise ValueError("runtime must be sglang or vllm-omni") + transport = _TRANSPORTS.get((plan["model_id"], runtime)) + if transport is None: + raise ValueError(f"{plan['model_id']} on {runtime} is not an audited transport") + return transport + + +def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: + """Validate the supported frozen H3 T2VA or Wan T2V plan without requests. + + This is the execution safety contract; the broader study/registry schemas + remain separate. Unknown root metadata is preserved in the plan digest. + """ + if not isinstance(plan, dict): + raise ValueError("plan must be an object") + try: + frozen = json.loads(canonical_json_bytes(plan)) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("plan must contain finite JSON values") from exc + for field in ("plan_id", "model_id", "model_revision"): + _text(frozen.get(field), field) + if frozen["model_id"] not in {MODEL_ID, WAN_MODEL_ID}: + raise ValueError(f"this MVP supports only {MODEL_ID} and {WAN_MODEL_ID}") + if not re.fullmatch(r"[a-fA-F0-9]{40}", frozen["model_revision"]): + raise ValueError("model_revision must be an immutable 40-character commit") + generation = frozen.get("generation") + if frozen["model_id"] == WAN_MODEL_ID: + _validate_wan_generation(generation) + else: + _validate_h3_generation(generation) cases = frozen.get("cases") if not isinstance(cases, list) or not cases: raise ValueError("cases must be a nonempty array") @@ -162,6 +220,8 @@ def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: for field in ("requires_motion", "requires_sound"): if not isinstance(case.get(field), bool): raise ValueError(f"case.{field} must be an explicit boolean") + if frozen["model_id"] == WAN_MODEL_ID and any(case["requires_sound"] for case in cases): + raise ValueError("Wan is video-only; requires_sound must be false") _positive(frozen.get("repetitions"), "repetitions", maximum=MAX_SLOTS, integer=True) warmups = frozen.get("warmup_runs") if isinstance(warmups, bool) or not isinstance(warmups, int) or warmups < 0: @@ -185,6 +245,11 @@ def _slots(plan: dict[str, Any]) -> list[dict[str, Any]]: def _payload(plan: dict[str, Any], slot: dict[str, Any], runtime: str) -> dict[str, Any]: generation = plan["generation"] + if plan["model_id"] == WAN_MODEL_ID: + _transport(plan, runtime) + return {"model": plan["model_id"], "prompt": slot["prompt"], "seed": slot["seed"], + **{key: value for key, value in generation.items() if key != "frame_count"}, + "num_frames": generation["frame_count"]} target = { "duration_seconds": generation["duration_seconds"], "aspect_ratio": generation["aspect_ratio"], @@ -214,6 +279,7 @@ def preview_plan(plan: dict[str, Any], *, runtime: str = "sglang") -> dict[str, if runtime not in RUNTIMES: raise ValueError("runtime must be sglang or vllm-omni") frozen = validate_plan(plan) + _transport(frozen, runtime) return { "evidence_kind": "request_preview_no_generation", "plan_id": frozen["plan_id"], "plan_sha256": _digest(frozen), "runtime": runtime, @@ -500,8 +566,7 @@ def run_plan( """ frozen = validate_plan(plan) serving = serving_settings(serving_concurrency, delivery_deadline_seconds) - if runtime not in RUNTIMES: - raise ValueError("runtime must be sglang or vllm-omni") + transport = _transport(frozen, runtime) for field, value in (("runtime_revision", runtime_revision), ("hardware_label", hardware_label), ("model_revision", model_revision)): _text(value, field) if not re.fullmatch(r"[a-fA-F0-9]{40}", runtime_revision): @@ -546,7 +611,7 @@ def run_plan( "hardware_label": hardware_label, "model_id": frozen["model_id"], "model_revision": model_revision, "endpoint": safe_endpoint, "identity_verification": "operator_declared", "plan_sha256": plan_sha256, - "protocol_source": _PROTOCOL_SOURCES[runtime], + "protocol_source": transport, "client_source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "media_evaluator": media_evaluator, "server_identity_caveat": "Model weights, runtime revision, hardware and server launch flags are not remotely verified.", @@ -568,6 +633,8 @@ def run_plan( "deadline": "network watchdog plus cooperative media deadline; one native decoder call may overrun", }, } + if frozen["model_id"] == WAN_MODEL_ID: + configuration["capabilities"] = {"audio_generation": "unsupported", "quality_qualification": "unsupported"} if serving: from . import mvp_serving configuration["serving"] = serving @@ -578,11 +645,12 @@ def run_plan( _write_json(directory / "configuration.json", configuration, exclusive=True) slots = _slots(frozen) scheduled = len(frozen["cases"]) * frozen["repetitions"] + model_label = "Wan" if frozen["model_id"] == WAN_MODEL_ID else "H3" run: dict[str, Any] = { - "bundle_version": "0.1.0", "bundle_type": "mvp_run", "run_id": "h3-" + uuid.uuid4().hex, + "bundle_version": "0.1.0", "bundle_type": "mvp_run", "run_id": ("wan22-" if frozen["model_id"] == WAN_MODEL_ID else "h3-") + uuid.uuid4().hex, "plan_id": frozen["plan_id"], "plan_sha256": plan_sha256, "plan": frozen, "configuration": configuration, "evidence_kind": "operator_endpoint", - "evidence_caveat": "Operator-managed HTTP endpoint; the client alone cannot establish H3 execution. Model identity is operator-declared, not attested. Mock-server tests are not H3 evidence. Controlled GPU execution requires a separate verified supervisor receipt.", + "evidence_caveat": f"Operator-managed HTTP endpoint; the client alone cannot establish {model_label} execution. Model identity is operator-declared, not attested. Mock-server tests are not {model_label} evidence. Controlled GPU execution requires a separate verified supervisor receipt.", "started_at": _timestamp(), "finished_at": None, "status": "partial", "measurement": {"boundary": "submit_to_validated_media", "concurrency": 1, "warmup_runs": frozen["warmup_runs"], "wall_seconds": 0.0}, @@ -611,7 +679,9 @@ def validate_media(record: dict, deadline: float) -> None: def attempt(slot: dict, *, defer_validation: bool = False) -> dict: nonlocal abort_reason, interrupted expected = {**frozen["generation"], "requires_motion": slot["requires_motion"], - "requires_sound": slot["requires_sound"], "audio_required": True} + "requires_sound": slot["requires_sound"], "audio_required": frozen["model_id"] == MODEL_ID} + if frozen["model_id"] == WAN_MODEL_ID: + expected["audio_forbidden"] = True expected["duration_seconds"] = expected["frame_count"] / expected["fps"] record: dict[str, Any] = { **{key: slot[key] for key in ("slot_id", "case_id", "prompt", "seed", "repetition", "phase")}, diff --git a/experimental/video-generation/evaluator/mvp_serving_smoke.py b/experimental/video-generation/evaluator/mvp_serving_smoke.py index 6bf633d154..089b942a25 100644 --- a/experimental/video-generation/evaluator/mvp_serving_smoke.py +++ b/experimental/video-generation/evaluator/mvp_serving_smoke.py @@ -13,7 +13,7 @@ from .mvp_gpu_evidence import verify_measurement_job from .mvp_report import _CSS, _number from .mvp_serving import settings -from .mvp_runner import _summary +from .mvp_runner import WAN_MODEL_ID, _summary CONCURRENCIES = (1, 2, 4) @@ -35,6 +35,7 @@ def validate_spec(spec: dict) -> dict: def _report(root: Path, matrix: dict) -> None: + title = "Wan2.2 serving smoke" if matrix["plan"]["model_id"] == WAN_MODEL_ID else "H3 serving smoke" rows, media = [], [] for cell in matrix["cells"]: summary = cell["completion"] @@ -56,7 +57,7 @@ def _report(root: Path, matrix: dict) -> None: report.mkdir(exist_ok=True) (report / "index.html").write_text( '' - f'H3 serving smoke

H3 serving smoke

' + f'{title}

{title}

' f'

One hardware configuration; {matrix["requests_per_configuration"]} measured requests at each concurrency. Warmups are separate. ' 'Latency is submit → downloaded media for technically valid clips. Throughput is valid clips / delivery wall seconds. ' 'Percentiles from small samples are preliminary and do not establish sustainable capacity. Failed and unstarted requests remain counted.

' @@ -76,7 +77,7 @@ def run_matrix(spec: dict, root: Path, *, concurrencies=CONCURRENCIES) -> dict: concurrencies = validate_concurrencies(concurrencies) count = len(spec["plan"]["cases"]) * spec["plan"]["repetitions"] deadline = time.monotonic() + spec["limits"]["job_seconds"] - matrix = {"schema_version": "1.0.0", "bundle_type": "h3_serving_smoke_matrix", "status": "running", + matrix = {"schema_version": "1.0.0", "bundle_type": "video_serving_smoke_matrix" if spec["plan"]["model_id"] == WAN_MODEL_ID else "h3_serving_smoke_matrix", "status": "running", "started_at": gpu._now(), "plan": spec["plan"], "runtime": spec["baseline"], "server": spec["server"], "gpu_uuids": spec["gpu_uuids"], "scheduled": count * len(concurrencies), "requests_per_configuration": count, "warmup_per_configuration": spec["plan"]["warmup_runs"], diff --git a/experimental/video-generation/evaluator/mvp_srt_selected.py b/experimental/video-generation/evaluator/mvp_srt_selected.py index c168e4002c..32f413057d 100644 --- a/experimental/video-generation/evaluator/mvp_srt_selected.py +++ b/experimental/video-generation/evaluator/mvp_srt_selected.py @@ -9,7 +9,7 @@ from specs import h3_spec -from .mvp_runner import run_plan, validate_plan +from .mvp_runner import WAN_MODEL_ID, run_plan, validate_plan from .mvp_srt import BENCHMARK_STEM, CELL_DIR, _write, benchmark_window @@ -37,23 +37,46 @@ def h3_vllm_model_view(snapshot: Path, output: Path) -> Path: return output -def server_argv(plan: dict, runtime: str, model_path: Path, port: int, gpus: int) -> list[str]: - if plan["model_id"] != "MiniMaxAI/MiniMax-H3" or gpus != 2: - raise ValueError("this H3 native profile requires two GPUs") +def server_argv( + plan: dict, runtime: str, model_path: Path, port: int, gpus: int, + *, output: Path | None = None, +) -> list[str]: + wan = plan["model_id"] == WAN_MODEL_ID + if (wan and gpus != 8) or (not wan and (plan["model_id"] != "MiniMaxAI/MiniMax-H3" or gpus != 2)): + raise ValueError("the native video profiles require two GPUs for H3 or eight Wan GPUs") if runtime == "sglang": from .mvp_gpu_job import _server_argv + if wan and output is None: + raise ValueError("Wan SGLang requires a job-owned output path") spec = { "baseline": {"python": "/usr/bin/python3"}, "model": {"path": str(model_path), "revision": plan["model_revision"]}, "plan": plan, - "server": {"ulysses_degree": 2, "tp_size": 1, "encoder_parallel": "auto", - "performance_mode": "speed", "dit_cpu_offload": False}, + "server": ( + {"ulysses_degree": 8, "tp_size": 1, "dit_cpu_offload": False, + "dit_layerwise_offload": False, "text_encoder_cpu_offload": False, + "vae_cpu_offload": False} + if wan else + {"ulysses_degree": 2, "tp_size": 1, "encoder_parallel": "auto", + "performance_mode": "speed", "dit_cpu_offload": False} + ), "gpu_uuids": ["assigned"] * gpus, "port": port, } - return _server_argv(spec, "baseline", host="0.0.0.0") + return _server_argv( + spec, "baseline", host="0.0.0.0", + output_path=str(output / "generated") if wan else None, + ) if runtime == "vllm-omni": + if wan: + return [ + "vllm", "serve", str(model_path), "--omni", + "--served-model-name", plan["model_id"], "--host", "0.0.0.0", "--port", str(port), + "--use-hsdp", "--cfg-parallel-size", "2", "--usp", "4", + "--vae-patch-parallel-size", "8", "--vae-use-tiling", + "--init-timeout", "3600", "--stage-init-timeout", "3600", + ] return [ "vllm", "serve", str(model_path), "--served-model-name", plan["model_id"], "--omni", "--host", "0.0.0.0", "--port", str(port), @@ -73,9 +96,11 @@ def serve( raise ValueError("runtime revision must be an immutable commit") if not model_path.is_dir(): raise ValueError("the pinned model snapshot is unavailable") - if runtime == "vllm-omni": + if runtime == "vllm-omni" and plan["model_id"] == "MiniMaxAI/MiniMax-H3": model_path = h3_vllm_model_view(model_path, output / "model-view") - argv = server_argv(plan, runtime, model_path, port, gpus) + if plan["model_id"] == WAN_MODEL_ID and runtime == "sglang": + (output / "generated").mkdir(parents=True, exist_ok=True) + argv = server_argv(plan, runtime, model_path, port, gpus, output=output) from .mvp_gpu_job import cuda_devices uuids = cuda_devices() diff --git a/experimental/video-generation/export_ci.py b/experimental/video-generation/export_ci.py index 45f970dccb..978211db32 100644 --- a/experimental/video-generation/export_ci.py +++ b/experimental/video-generation/export_ci.py @@ -178,7 +178,7 @@ def publish(run_ids: list[str], output: Path, hardware: Path | None, *, hardware files = files_with_nested_seals(target) (target / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in sorted(files.items()))) ci.write(output / "index.json", catalog) - for name in ("result.schema.json", "RESULTS.md", "RESULTS_zh.md"): + for name in ("result.schema.json", "RESULTS.md"): shutil.copyfile(Path(__file__).parent / name, output / name) (output / "SHA256SUMS").write_text("".join(f"{digest} {name}\n" for name, digest in files_with_nested_seals(output).items())) return exit_code diff --git a/experimental/video-generation/mvp/h3-startup-pair.plan.json b/experimental/video-generation/mvp/h3-startup-pair.plan.json new file mode 100644 index 0000000000..6eac97031b --- /dev/null +++ b/experimental/video-generation/mvp/h3-startup-pair.plan.json @@ -0,0 +1,29 @@ +{ + "plan_id": "h3-fl2va-t2va-startup-pair-v1", + "model_id": "MiniMaxAI/MiniMax-H3", + "model_revision": "42ed227ee7df40d41602854ae760620d6eb651fe", + "generation": { + "duration_seconds": 4, + "aspect_ratio": "16:9", + "width": 1344, + "height": 768, + "frame_count": 107, + "fps": 24, + "audio_sample_rate_hz": 32000, + "audio_channels": 2, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0 + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a quiet rehearsal room. The drumsticks visibly contact the drumhead on each tap. The three crisp snare hits and the natural room ambience are audible. The camera remains still. No music, speech, subtitles, or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": true + } + ], + "repetitions": 1, + "warmup_runs": 1 +} diff --git a/experimental/video-generation/mvp/wan22-gpu-job.template.json b/experimental/video-generation/mvp/wan22-gpu-job.template.json new file mode 100644 index 0000000000..776c35b0b4 --- /dev/null +++ b/experimental/video-generation/mvp/wan22-gpu-job.template.json @@ -0,0 +1,107 @@ +{ + "schema_version": "0.1.0", + "job_id": "wan22-serving-REPLACE_WITH_UNIQUE_ID", + "authorization": { + "compute_approved": false, + "model_license_reviewed": false, + "approval_reference": "" + }, + "allocation": { + "mode": "cooperative_shared", + "label": "REPLACE_WITH_ACTUAL_APPROVED_ALLOCATION" + }, + "gpu_uuids": [ + "REPLACE_WITH_APPROVED_FULL_GPU_UUID_1" + ], + "port": 30283, + "lock_directory": "/REPLACE_WITH_OWNED_PRIVATE_LOCK_DIRECTORY", + "baseline": { + "python": "/REPLACE_WITH_BASELINE_ENV/bin/python", + "source": "/REPLACE_WITH_BASELINE_SOURCE", + "revision": "REPLACE_WITH_MANIFEST_REVISION", + "source_sha256": "REPLACE_WITH_MANIFEST_SOURCE_SHA256" + }, + "candidate": { + "python": "/REPLACE_WITH_CANDIDATE_ENV/bin/python", + "source": "/REPLACE_WITH_CANDIDATE_SOURCE", + "revision": "REPLACE_WITH_MANIFEST_REVISION", + "source_sha256": "REPLACE_WITH_MANIFEST_SOURCE_SHA256" + }, + "model": { + "path": "/REPLACE_WITH_VERIFIED_WAN_DIFFUSERS_SNAPSHOT", + "revision": "5be7df9619b54f4e2667b2755bc6a756675b5cd7", + "files": [] + }, + "server": { + "ulysses_degree": 1, + "tp_size": 1, + "dit_cpu_offload": true, + "dit_layerwise_offload": false, + "text_encoder_cpu_offload": true, + "vae_cpu_offload": true + }, + "plan": { + "plan_id": "wan22-t2v-a14b-serving-smoke-v1", + "model_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "model_revision": "5be7df9619b54f4e2667b2755bc6a756675b5cd7", + "generation": { + "width": 832, + "height": 480, + "frame_count": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "" + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a rehearsal room. The drumsticks visibly contact the drumhead on each tap. The camera remains still. No subtitles or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "pouring-water", + "prompt": "A single continuous close-up of a hand steadily pouring water from a clear glass jug into an empty drinking glass on a wooden table. The water level visibly rises. Daylight, a still camera, no subtitles or scene cuts.", + "seed": 23, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "bicycle-pass", + "prompt": "A single continuous wide shot of a cyclist riding from the left side of the frame to the right along a park path. The moving wheels remain visible. The camera stays still. No subtitles or scene cuts.", + "seed": 37, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "footsteps", + "prompt": "A single continuous low-angle shot of a person in ordinary shoes walking across a wooden floor from left to right. Each step visibly lands on the floor. The camera stays still. No subtitles or scene cuts.", + "seed": 53, + "requires_motion": true, + "requires_sound": false + } + ], + "repetitions": 1, + "warmup_runs": 1 + }, + "policy": { + "policy_id": "wan22-no-quality-qualification-v1", + "calibration_status": "uncalibrated" + }, + "limits": { + "job_seconds": 4500, + "startup_seconds": 900, + "request_seconds": 900, + "cleanup_seconds": 60, + "telemetry_interval_seconds": 1, + "command_seconds": 10, + "max_idle_memory_mib": 64 + }, + "serving": { + "concurrency": 1 + } +} diff --git a/experimental/video-generation/mvp/wan22-serving.plan.json b/experimental/video-generation/mvp/wan22-serving.plan.json new file mode 100644 index 0000000000..afaa649360 --- /dev/null +++ b/experimental/video-generation/mvp/wan22-serving.plan.json @@ -0,0 +1,48 @@ +{ + "plan_id": "wan22-t2v-a14b-serving-smoke-v1", + "model_id": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "model_revision": "5be7df9619b54f4e2667b2755bc6a756675b5cd7", + "generation": { + "width": 832, + "height": 480, + "frame_count": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "" + }, + "cases": [ + { + "case_id": "drum-taps", + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a rehearsal room. The drumsticks visibly contact the drumhead on each tap. The camera remains still. No subtitles or scene cuts.", + "seed": 11, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "pouring-water", + "prompt": "A single continuous close-up of a hand steadily pouring water from a clear glass jug into an empty drinking glass on a wooden table. The water level visibly rises. Daylight, a still camera, no subtitles or scene cuts.", + "seed": 23, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "bicycle-pass", + "prompt": "A single continuous wide shot of a cyclist riding from the left side of the frame to the right along a park path. The moving wheels remain visible. The camera stays still. No subtitles or scene cuts.", + "seed": 37, + "requires_motion": true, + "requires_sound": false + }, + { + "case_id": "footsteps", + "prompt": "A single continuous low-angle shot of a person in ordinary shoes walking across a wooden floor from left to right. Each step visibly lands on the floor. The camera stays still. No subtitles or scene cuts.", + "seed": 53, + "requires_motion": true, + "requires_sound": false + } + ], + "repetitions": 1, + "warmup_runs": 1 +} diff --git a/experimental/video-generation/pyproject.toml b/experimental/video-generation/pyproject.toml index dbabe2bc1c..d0b6bd7429 100644 --- a/experimental/video-generation/pyproject.toml +++ b/experimental/video-generation/pyproject.toml @@ -10,7 +10,7 @@ requires-python = ">=3.12" dependencies = ["av==16.1.0", "numpy==2.3.5", "pyyaml==6.0.3"] [project.optional-dependencies] -dev = ["pytest==8.4.2"] +dev = ["pytest==8.4.2", "jsonschema>=4,<5", "rfc3339-validator==0.1.4"] [project.scripts] vgbench = "evaluator.cli:main" @@ -21,4 +21,4 @@ include = ["evaluator*"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = [".", "tests"] +pythonpath = [".", "tests", "specs"] diff --git a/experimental/video-generation/specs/README.md b/experimental/video-generation/specs/README.md index d6f4be5fda..552c10abc4 100644 --- a/experimental/video-generation/specs/README.md +++ b/experimental/video-generation/specs/README.md @@ -1,8 +1,14 @@ -# H3 workload spec +# Video workload specs + +**English** | [中文](README_zh.md) + +## MiniMax-H3 `h3.yaml` defines the MiniMax-H3 text-to-video-and-audio workload that SGLang and vLLM-Omni both run. `h3_spec.py` loads and validates it, then builds the plan object that the runner from PR #2916 executes and hashes. -`h3-vllm-omni-serving.md` records the pinned vLLM-Omni image, server command, parallelism and attention backend that served this workload on 2x H100. +The H3 spec is an explicit dependency of the shared loader's default command. +`h3-vllm-omni-serving.md` records the pinned vLLM-Omni H3 image and two-GPU server command. +The `release-v1` set crosses the four existing prompts with five recorded seeds for 20 measured clips. ## What the YAML records @@ -35,9 +41,18 @@ It serializes with sorted keys, compact separators, UTF-8, and no NaN. From the repository root: ```bash -uv run --no-project --python 3.12 --with 'pyyaml>=6,<7' --with 'pytest>=8,<9' python -m pytest experimental/video-generation/tests/test_h3_spec.py -q +uv run --no-project --python 3.12 --with 'pyyaml>=6,<7' --with 'pytest>=8,<9' python -m pytest experimental/video-generation/tests/test_h3_spec.py experimental/video-generation/tests/test_wan_spec.py -q uv run --no-project --python 3.12 --with 'pyyaml>=6,<7' python experimental/video-generation/specs/h3_spec.py ``` The second command prints `spec_sha256` and each set's `plan_id` and `plan_sha256` as JSON. To hash another spec file, pass its path as the argument. + +## Wan2.2-T2V-A14B + +`wan.yaml` defines the Wan2.2-T2V-A14B text-to-video workload in the same schema and is loaded by the same `h3_spec.py`. +Its `frozen-3733` set reproduces PR #3733's `wan22-t2v-a14b-serving-smoke-v1` plan, whose hash is `1f1a093b391bfd94f64d6e492948989850508ac4dad472eeaa98785fd26327d6`; its `shared-v1` set runs every #3733 prompt with its #3733 seed and a second seed 100 higher (the second seeds are new, so each cites the #3733 line of the seed it derives from). +The `release-v1` set crosses the four recorded prompts with five cited seeds for 20 measured clips and one warmup on each engine. +Both engines receive every generation field (`frame_count` travels as `num_frames`); width, height, frame count and fps are checked on the clip, and the video-only model contract adds the audio-absence check. +`wan-vllm-omni-serving.md` records the pinned vLLM-Omni image, server command and parallelism that served this workload on 8x H100, and `../tests/fixtures/wan-vllm-omni-a8576ccb/` holds the request, server record and ffprobe from that run. +The Wan tests live in `tests/test_wan_spec.py` and `tests/test_wan_runner.py`; `.github/workflows/test-h3-video.yml` runs the whole `experimental/video-generation` suite on draft and ready pull requests. diff --git a/experimental/video-generation/specs/README_zh.md b/experimental/video-generation/specs/README_zh.md new file mode 100644 index 0000000000..46acc7c5d1 --- /dev/null +++ b/experimental/video-generation/specs/README_zh.md @@ -0,0 +1,44 @@ +# 视频工作负载规格 + +[English](README.md) | **中文** + +## MiniMax-H3 + +`h3.yaml` 定义 SGLang 与 vLLM-Omni 共用的 MiniMax-H3 文生视频及音频工作负载。 +`h3_spec.py` 加载、验证规格,并生成 #2916 runner 执行及计算哈希的 plan 对象。 +此处包含 H3 规格,因为共享 loader 的默认命令使用它。 + +## YAML 字段 + +- `model` 和每个 `generation` 字段都有 `value` 与 `@::` 格式的 `source` 引用。 +- `sent.sglang` 和 `sent.vllm-omni` 表示相应引擎的请求是否携带该字段。SGLang 不接受调用方设置的 FPS 和帧数,固定以 24 FPS 运行,并根据 `target` 推导帧数和分辨率。只有 vLLM-Omni 接收 FPS、帧数、宽度和高度。 +- `checked_on_clip` 表示是否用该字段核对生成的视频。runner 使用 `frame_count / fps` 检查时长,因此不直接检查 `duration_seconds`。H3 的 107 帧、24 FPS 对应 4.458 秒。 +- `prompts` 定义提示词及动态、声音要求。`sets` 中的 case 将提示词与 seed 配对,`seed_sources` 为每个 seed 提供引用。 +- `reference` 记录 SGLang 参考运行,只供查阅,不参与读取或哈希。该运行使用 workflow commit `f7c7f58`;其中的 spec、`mvp_runner.py` 和 `mvp_media.py` 与 `0cb4c847d` 逐字节一致。 + +## 哈希约定 + +`canonical_json_bytes` 与 #2916 `evaluator/mvp_runner.py` 中的实现逐字节一致。它按键排序、使用紧凑分隔符和 UTF-8,不允许 NaN。 + +- `plan_sha256(plan_for(spec, set_id))` 与 #2916 runner 对同一 plan 记录的 `plan_sha256` 相同。`frozen-c124` 集合还原 #2916 的 `h3-serving-2gpu-c124-v1` plan,哈希为 `8d51a472caa7db0f5d4c33adec1e9c9f852e7144cf25356085802edaf7d80d68`。 +- `spec_sha256(spec)` 覆盖模型、generation 值,以及每个集合解析后的 case 与计数。 +- 引用、`sent`、`checked_on_clip`、`reference`、YAML 键顺序、注释和排版均不进入两个哈希。GPU UUID、job ID、端口和主机名也不进入哈希。 +- `plan_for(spec, set_id, repetitions=..., warmup_runs=...)` 可覆盖计数。如果计数与集合原值不同,plan ID 会增加 `-r-w` 后缀,避免缩短运行与完整运行共用 ID 或哈希。 + +## 运行测试并打印哈希 + +从仓库根目录运行: + +```bash +uv run --no-project --python 3.12 --with 'pyyaml==6.0.3' --with 'pytest>=8,<9' python -m pytest experimental/video-generation/tests/test_wan_spec.py -q +uv run --no-project --python 3.12 --with 'pyyaml>=6,<7' python experimental/video-generation/specs/h3_spec.py +``` + +第二条命令以 JSON 输出 `spec_sha256`,以及每个集合的 `plan_id` 与 `plan_sha256`。传入另一个规格文件路径可以计算它的哈希。 + +## Wan2.2-T2V-A14B + +`wan.yaml` 使用相同 schema 定义 Wan2.2-T2V-A14B 纯视频工作负载,并由同一个 `h3_spec.py` 加载。`frozen-3733` 集合还原 #3733 的 `wan22-t2v-a14b-serving-smoke-v1` plan,哈希为 `1f1a093b391bfd94f64d6e492948989850508ac4dad472eeaa98785fd26327d6`。`shared-v1` 为 #3733 的每条提示词保留原 seed,并增加一个高 100 的 seed;每个新 seed 引用其原 seed 的来源行。`release-v1` 将四条已有提示词与五个已记录的 seed 交叉组合,每个引擎运行 20 条测量视频和一次预热。 + +两个引擎接收全部 generation 字段,其中 `frame_count` 作为 `num_frames` 发送。runner 检查视频宽度、高度、帧数和 FPS;纯视频模型约定还要求没有音频流。 +[Wan vLLM-Omni 服务记录](wan-vllm-omni-serving.md)保留八卡 H100 上的固定镜像、服务命令和并行配置。`../tests/fixtures/wan-vllm-omni-a8576ccb/` 保留该次请求、服务记录和 ffprobe 输出。Wan 测试位于 `tests/test_wan_spec.py` 和 `tests/test_wan_runner.py`;`.github/workflows/test-h3-video.yml` 在草稿及正式 PR 上运行整个 `experimental/video-generation` 测试集。 diff --git a/experimental/video-generation/specs/wan-vllm-omni-serving.md b/experimental/video-generation/specs/wan-vllm-omni-serving.md new file mode 100644 index 0000000000..65a9535d19 --- /dev/null +++ b/experimental/video-generation/specs/wan-vllm-omni-serving.md @@ -0,0 +1,80 @@ +# Serving Wan2.2-T2V-A14B on vLLM-Omni + +Configuration observed for one Wan2.2-T2V-A14B request on 8x H100 80GB on 2026-10-17 (dsxe, Slurm job 445). + +## Pins + +| Component | Value | +| --- | --- | +| vLLM-Omni | release v0.30.0, commit `a8576ccb725c4e21cd13c3eb5f9a546b21149d2b` (contains the upstream Wan2.2-T2V recipe's pinned `4c7a98c2`) | +| Image | `vllm/vllm-omni:v0.30.0`, manifest digest `sha256:fc77cfaac7b1b43c30cbf32eaea214864434c6e0f303e6e9eb9f64104241a994`, linux/amd64 `sha256:b09c9eab7d2e30817208de92ccb482756d13d4339377d7cc4c4ce9f47bf04174` | +| In-image versions | vllm 0.30.0, vllm-omni 0.30.0, torch 2.13.0+cu130, av 18.1.0, numpy 2.2.6 | +| Model | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` revision `5be7df9619b54f4e2667b2755bc6a756675b5cd7` (126 GB, 49 files; same as `specs/wan.yaml` and PR #3733) | + +## Server command + +``` +vllm serve --omni --served-model-name Wan-AI/Wan2.2-T2V-A14B-Diffusers \ + --host 127.0.0.1 --port 8091 \ + --use-hsdp --cfg-parallel-size 2 --usp 4 --vae-patch-parallel-size 8 --vae-use-tiling \ + --init-timeout 3600 --stage-init-timeout 3600 +``` + +`` is the Hugging Face cache snapshot directory for the revision above (serve the directory, not the Hub ID, under `HF_HUB_OFFLINE=1`). `--served-model-name` keeps the model name the evaluator sends. + +Environment: + +``` +HF_HUB_OFFLINE=1 +HF_HUB_CACHE= +VLLM_OMNI_SERVER_STORAGE__PATH=/server-storage +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +``` + +## Parallelism + +The upstream recipe's "official open-source model (with CFG)" layout: CFG parallel 2 x Ulysses sequence parallel 4 = 8 GPUs, HSDP weight sharding, VAE patch parallel 8 with tiling. The server resolved the `FLASH_ATTN` attention backend for both `self` and `cross` roles at startup. Both init timeouts must be raised: the 600 s default aborted the first cold start (job 443) while the eight workers were still reading the checkpoint; the warm-cache restart was ready after 574 s. + +## Request mapping + +`POST /v1/videos` is a multipart form at this pin; the field names come from `vllm_omni/entrypoints/openai/video/generation/helpers.py::_parse_video_form` and `vllm_omni/entrypoints/openai/protocol/videos.py::VideoGenerationRequest`. + +| spec field | form field | +| --- | --- | +| `model.id` | `model` | +| case prompt, case seed | `prompt`, `seed` | +| `generation.width`, `generation.height` | `width`, `height` | +| `generation.frame_count` | `num_frames` | +| `generation.fps` | `fps` | +| `generation.num_inference_steps` | `num_inference_steps` | +| `generation.guidance_scale`, `generation.guidance_scale_2` | `guidance_scale`, `guidance_scale_2` | +| `generation.flow_shift` | `flow_shift` (per-request scheduler shift; the server default is 5.0) | +| `generation.negative_prompt` | `negative_prompt` (FastAPI drops the empty field; `pipeline_wan2_2.py` encodes `""` for it) | + +`boundary_ratio` is not sent; both engines use the checkpoint's 0.875. The runner polls `GET /v1/videos/{id}` until `completed` and downloads `GET /v1/videos/{id}/content`. At this pin a failed job answers the poll with a non-2xx status, which the ported runner records as a transport error rather than `provider_failed` (the same holds for the H3 vLLM-Omni path in #3733). + +Runner call (the SGLang path is unchanged; only `runtime` differs): + +```python +from specs import h3_spec +from evaluator import mvp_runner + +spec = h3_spec.load_spec("specs/wan.yaml") +mvp_runner.run_plan( + h3_spec.plan_for(spec, "shared-v1"), + "", + endpoint="http://127.0.0.1:8091", + runtime="vllm-omni", + runtime_revision="a8576ccb725c4e21cd13c3eb5f9a546b21149d2b", + hardware_label="", + model_revision=spec["model"]["revision"]["value"], +) +``` + +## Observed on 8x H100 (informational, not a benchmark) + +- `/health` 200 after 574 s with a warm page cache; eight `DiffusionWorker_SP{0-3}_CFG{0,1}` processes, `peak_memory_mb` 25566 in the server's job record. +- One request of case `drum-taps-s11` (a one-case copy of `shared-v1` with `warmup_runs` 0): 40 denoise steps at 1.23 it/s, server `inference_time_s` 37.43, 37.97 s submit to validated clip. +- Clip: 832x480, 81 frames, 16 fps, 5.0625 s, h264, no audio stream, 2,225,797 bytes; every frame has pixel std 69 to 71 on the 0 to 255 scale, and frames 1, 41 and 81 show the prompted drummer. +- Power (nvidia-smi 1 Hz): the sum of per-GPU board means was 4518 W during generation, using 42 of 60 samples per GPU, against 543 W idle before launch. This is a single-request observation, not a PowerX curve or a matched power benchmark. +- Evidence: `tests/fixtures/wan-vllm-omni-a8576ccb/` holds the runner request, the server's job record and the ffprobe of the clip. diff --git a/experimental/video-generation/specs/wan.yaml b/experimental/video-generation/specs/wan.yaml new file mode 100644 index 0000000000..ac92ce2c37 --- /dev/null +++ b/experimental/video-generation/specs/wan.yaml @@ -0,0 +1,184 @@ +# Wan2.2-T2V-A14B text-to-video workload shared by SGLang and vLLM-Omni; load it with h3_spec.py. +# `sent` follows the Wan payload builder at +# feat/wan-serving-runner@77f2080ce:experimental/video-generation/evaluator/mvp_runner.py:227-240 +# (one dict for both engines; frame_count travels as num_frames). `checked_on_clip` follows the clip +# checks at feat/wan-serving-runner@77f2080ce:experimental/video-generation/evaluator/mvp_media.py:386-404 +# against the expected media built at mvp_runner.py:666-669. The video-only model contract adds +# audio.absent (mvp_media.py:409) from the model id, and requires_motion adds video.motion_presence +# (mvp_media.py:422); neither is a generation field. + +schema_version: 1 + +model: + id: + value: Wan-AI/Wan2.2-T2V-A14B-Diffusers + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:3 + revision: + value: 5be7df9619b54f4e2667b2755bc6a756675b5cd7 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:4 + +generation: + width: + value: 832 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:6 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: true + height: + value: 480 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:7 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: true + frame_count: + # Sent as num_frames on both engines. The clip duration check is derived: 81 / 16 = 5.0625 s. + value: 81 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:8 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: true + fps: + value: 16 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:9 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: true + num_inference_steps: + value: 40 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:10 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: false + guidance_scale: + value: 4.0 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:11 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: false + guidance_scale_2: + value: 3.0 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:12 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: false + flow_shift: + # validate_plan pins 12.0; vLLM-Omni a8576ccb applies the per-request value (pipeline_wan2_2.py:121-130). + value: 12.0 + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:13 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: false + negative_prompt: + # Empty on purpose. FastAPI drops the empty multipart field on vLLM-Omni a8576ccb and + # pipeline_wan2_2.py encodes "" for the missing value; SGLang receives the key in JSON. + value: "" + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:14 + sent: {sglang: true, vllm-omni: true} + checked_on_clip: false + +prompts: + drum-taps: + text: "A single continuous medium shot of a drummer tapping a snare drum three times in a rehearsal room. The drumsticks visibly contact the drumhead on each tap. The camera remains still. No subtitles or scene cuts." + requires_motion: true + requires_sound: false + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:19 + pouring-water: + text: "A single continuous close-up of a hand steadily pouring water from a clear glass jug into an empty drinking glass on a wooden table. The water level visibly rises. Daylight, a still camera, no subtitles or scene cuts." + requires_motion: true + requires_sound: false + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:26 + bicycle-pass: + text: "A single continuous wide shot of a cyclist riding from the left side of the frame to the right along a park path. The moving wheels remain visible. The camera stays still. No subtitles or scene cuts." + requires_motion: true + requires_sound: false + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:33 + footsteps: + text: "A single continuous low-angle shot of a person in ordinary shoes walking across a wooden floor from left to right. Each step visibly lands on the floor. The camera stays still. No subtitles or scene cuts." + requires_motion: true + requires_sound: false + source: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:40 + +sets: + frozen-3733: + # #3733's own plan: one seed per prompt, case ids without a seed suffix, plan_sha256 + # 1f1a093b391bfd94f64d6e492948989850508ac4dad472eeaa98785fd26327d6. + plan_id: wan22-t2v-a14b-serving-smoke-v1 + repetitions: 1 + warmup_runs: 1 + seed_sources: + 11: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:20 + 23: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:27 + 37: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:34 + 53: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:41 + cases: + - {case_id: drum-taps, prompt: drum-taps, seed: 11} + - {case_id: pouring-water, prompt: pouring-water, seed: 23} + - {case_id: bicycle-pass, prompt: bicycle-pass, seed: 37} + - {case_id: footsteps, prompt: footsteps, seed: 53} + shared-v1: + # The shared set uses every #3733 prompt with its recorded seed and a second seed 100 higher. + # The second seeds are new here, so each cites the #3733 line of the seed it is derived from. + plan_id: wan22-t2v-a14b-shared-v1 + repetitions: 1 + warmup_runs: 1 + seed_sources: + 11: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:20 + 111: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:20 + 23: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:27 + 123: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:27 + 37: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:34 + 137: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:34 + 53: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:41 + 153: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:41 + cases: + - {case_id: drum-taps-s11, prompt: drum-taps, seed: 11} + - {case_id: drum-taps-s111, prompt: drum-taps, seed: 111} + - {case_id: pouring-water-s23, prompt: pouring-water, seed: 23} + - {case_id: pouring-water-s123, prompt: pouring-water, seed: 123} + - {case_id: bicycle-pass-s37, prompt: bicycle-pass, seed: 37} + - {case_id: bicycle-pass-s137, prompt: bicycle-pass, seed: 137} + - {case_id: footsteps-s53, prompt: footsteps, seed: 53} + - {case_id: footsteps-s153, prompt: footsteps, seed: 153} + release-v1: + # Twenty measured inputs: four recorded prompts crossed with five seeds already cited above. + plan_id: wan22-native-engine-release-v1 + repetitions: 1 + warmup_runs: 1 + seed_sources: + 11: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:20 + 23: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:27 + 37: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:34 + 53: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:41 + 111: feat/wan-serving-runner@77f2080ce:experimental/video-generation/mvp/wan22-serving.plan.json:20 + cases: + - {case_id: drum-taps-s11, prompt: drum-taps, seed: 11} + - {case_id: drum-taps-s23, prompt: drum-taps, seed: 23} + - {case_id: drum-taps-s37, prompt: drum-taps, seed: 37} + - {case_id: drum-taps-s53, prompt: drum-taps, seed: 53} + - {case_id: drum-taps-s111, prompt: drum-taps, seed: 111} + - {case_id: pouring-water-s11, prompt: pouring-water, seed: 11} + - {case_id: pouring-water-s23, prompt: pouring-water, seed: 23} + - {case_id: pouring-water-s37, prompt: pouring-water, seed: 37} + - {case_id: pouring-water-s53, prompt: pouring-water, seed: 53} + - {case_id: pouring-water-s111, prompt: pouring-water, seed: 111} + - {case_id: bicycle-pass-s11, prompt: bicycle-pass, seed: 11} + - {case_id: bicycle-pass-s23, prompt: bicycle-pass, seed: 23} + - {case_id: bicycle-pass-s37, prompt: bicycle-pass, seed: 37} + - {case_id: bicycle-pass-s53, prompt: bicycle-pass, seed: 53} + - {case_id: bicycle-pass-s111, prompt: bicycle-pass, seed: 111} + - {case_id: footsteps-s11, prompt: footsteps, seed: 11} + - {case_id: footsteps-s23, prompt: footsteps, seed: 23} + - {case_id: footsteps-s37, prompt: footsteps, seed: 37} + - {case_id: footsteps-s53, prompt: footsteps, seed: 53} + - {case_id: footsteps-s111, prompt: footsteps, seed: 111} + +# Informational only; not hashed. +reference: + sglang_3733: + pull_request: https://github.com/SemiAnalysisAI/InferenceX/pull/3733 + head: 77f2080ce + sglang_revision: 71de97b264b04dcd514cf904003028aefe9775c8 + plan_id: wan22-t2v-a14b-serving-smoke-v1 + plan_sha256: 1f1a093b391bfd94f64d6e492948989850508ac4dad472eeaa98785fd26327d6 + vllm_omni_h100: + serving_notes: wan-vllm-omni-serving.md + vllm_omni_revision: a8576ccb725c4e21cd13c3eb5f9a546b21149d2b + hardware: 8x NVIDIA H100 80GB HBM3 (dsxe, Slurm job 445, 2026-10-17) + plan_id: wan22-t2v-a14b-shared-v1 + plan_sha256: e80434fcec2c322274fd743ca0c3e25e90be30eae2b5d98111644f2b0295db3f + outcome: >- + One request of case drum-taps-s11 (a one-case copy of shared-v1 with warmup_runs 0) returned a + valid 832x480, 81-frame, 16 fps clip 38.0 s after submission; the request, the server's job + record and the ffprobe are in tests/fixtures/wan-vllm-omni-a8576ccb. diff --git a/experimental/video-generation/tests/fixtures/deployment-frontier.json b/experimental/video-generation/tests/fixtures/deployment-frontier.json new file mode 100644 index 0000000000..789d7f33ee --- /dev/null +++ b/experimental/video-generation/tests/fixtures/deployment-frontier.json @@ -0,0 +1,52 @@ +{ + "description": "Synthetic dominance cases shared with InferenceX-app; not measured H3 evidence.", + "cases": [ + { + "name": "lower-higher preserves ties and excludes dominated coordinates", + "xBetter": "lower", + "yBetter": "higher", + "points": [ + { "id": "slow", "x": 400, "y": 9 }, + { "id": "dominated", "x": 151, "y": 4.9 }, + { "id": "fast", "x": 78, "y": 6.66 }, + { "id": "tie", "x": 78, "y": 6.66 }, + { "id": "same-x", "x": 78, "y": 6 }, + { "id": "same-y", "x": 90, "y": 6.66 } + ], + "expected": ["fast", "tie", "slow"] + }, + { + "name": "higher-higher trades speed for efficiency", + "xBetter": "higher", + "yBetter": "higher", + "points": [ + { "id": "fast", "x": 2.46, "y": 6.66 }, + { "id": "dominated", "x": 1.27, "y": 4.9 }, + { "id": "efficient", "x": 1.2, "y": 7 } + ], + "expected": ["efficient", "fast"] + }, + { + "name": "lower-lower trades latency for cost", + "xBetter": "lower", + "yBetter": "lower", + "points": [ + { "id": "fast", "x": 78, "y": 0.15 }, + { "id": "dominated", "x": 151, "y": 0.204 }, + { "id": "cheap", "x": 300, "y": 0.1 } + ], + "expected": ["fast", "cheap"] + }, + { + "name": "higher-lower trades capacity for energy", + "xBetter": "higher", + "yBetter": "lower", + "points": [ + { "id": "fast", "x": 8, "y": 300 }, + { "id": "dominated", "x": 6, "y": 400 }, + { "id": "efficient", "x": 4, "y": 200 } + ], + "expected": ["efficient", "fast"] + } + ] +} diff --git a/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/ffprobe.json b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/ffprobe.json new file mode 100644 index 0000000000..04cc8e2052 --- /dev/null +++ b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/ffprobe.json @@ -0,0 +1,83 @@ +{ + "streams": [ + { + "index": 0, + "codec_name": "h264", + "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", + "profile": "Constrained Baseline", + "codec_type": "video", + "codec_tag_string": "avc1", + "codec_tag": "0x31637661", + "width": 832, + "height": 480, + "coded_width": 832, + "coded_height": 480, + "has_b_frames": 0, + "pix_fmt": "yuv420p", + "level": 30, + "chroma_location": "left", + "field_order": "progressive", + "refs": 1, + "is_avc": "true", + "nal_length_size": "4", + "id": "0x1", + "r_frame_rate": "16/1", + "avg_frame_rate": "16/1", + "time_base": "1/16384", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 82944, + "duration": "5.062500", + "bit_rate": "3515425", + "bits_per_raw_sample": "8", + "nb_frames": "81", + "nb_read_frames": "81", + "extradata_size": 39, + "disposition": { + "default": 1, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "non_diegetic": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0, + "multilayer": 0 + }, + "tags": { + "language": "und", + "handler_name": "VideoHandler", + "vendor_id": "[0][0][0][0]" + } + } + ], + "format": { + "filename": "/work/results/wan-vllm-omni-feas-20261017-a01/job-445/runner/artifacts/measurement-r001-c001.mp4", + "nb_streams": 1, + "nb_programs": 0, + "nb_stream_groups": 0, + "format_name": "mov,mp4,m4a,3gp,3g2,mj2", + "format_long_name": "QuickTime / MOV", + "start_time": "0.000000", + "duration": "5.062500", + "size": "2225797", + "bit_rate": "3517308", + "probe_score": 100, + "tags": { + "major_brand": "isom", + "minor_version": "512", + "compatible_brands": "isomiso2avc1mp41", + "encoder": "Lavf62.12.102" + } + } +} diff --git a/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/request.json b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/request.json new file mode 100644 index 0000000000..d01c6a326a --- /dev/null +++ b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/request.json @@ -0,0 +1,20 @@ +{ + "case_id": "drum-taps-s11", + "plan_id": "wan22-t2v-a14b-shared-v1", + "request": { + "flow_shift": 12.0, + "fps": 16, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "height": 480, + "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "negative_prompt": "", + "num_frames": 81, + "num_inference_steps": 40, + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a rehearsal room. The drumsticks visibly contact the drumhead on each tap. The camera remains still. No subtitles or scene cuts.", + "seed": 11, + "width": 832 + }, + "slot_id": "measurement-r001-c001", + "spec_plan_sha256": "e80434fcec2c322274fd743ca0c3e25e90be30eae2b5d98111644f2b0295db3f" +} diff --git a/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/server-record.json b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/server-record.json new file mode 100644 index 0000000000..c21f9f346d --- /dev/null +++ b/experimental/video-generation/tests/fixtures/wan-vllm-omni-a8576ccb/server-record.json @@ -0,0 +1,29 @@ +{ + "action": null, + "completed_at": 1792220495, + "created_at": 1792220458, + "duration_s": 5.0625, + "error": null, + "expires_at": null, + "file_name": "video_gen_aa9dfad96e1e4fd9acf8f7ef1d3669ed.mp4", + "fps": 16.0, + "id": "video_gen_aa9dfad96e1e4fd9acf8f7ef1d3669ed", + "inference_time_s": 37.43411935213953, + "media_type": "video/mp4", + "metrics": null, + "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "num_frames": 81, + "object": "video", + "peak_memory_mb": 25566.0, + "progress": 100, + "prompt": "A single continuous medium shot of a drummer tapping a snare drum three times in a rehearsal room. The drumsticks visibly contact the drumhead on each tap. The camera remains still. No subtitles or scene cuts.", + "quality": "default", + "remixed_from_video_id": null, + "seconds": "4", + "size": null, + "stage_durations": { + "queue_wait_ms": 0.5811508744955063, + "stage_0_gen_ms": 37187.02578544617 + }, + "status": "completed" +} diff --git a/experimental/video-generation/tests/test_dashboard_observations_schema.py b/experimental/video-generation/tests/test_dashboard_observations_schema.py new file mode 100644 index 0000000000..cbca2399a9 --- /dev/null +++ b/experimental/video-generation/tests/test_dashboard_observations_schema.py @@ -0,0 +1,152 @@ +"""Validate the additive source-side projection, not benchmark qualification.""" + +import json +from copy import deepcopy +from pathlib import Path + +import jsonschema +import pytest + +SCHEMA = json.loads( + (Path(__file__).parents[1] / "dashboard-observations.schema.json").read_text() +) +VALIDATOR = jsonschema.Draft202012Validator( + SCHEMA, format_checker=jsonschema.FormatChecker() +) + + +def sidecar(): + return { + "schemaVersion": 1, + "sourceRunId": "123", + "sourceSha": "a" * 40, + "cells": { + "c1": { + "runSha256": "b" * 64, + "specSha256": "c" * 64, + "deployment": { + "ring": 1, + "cfg": 1, + "maxBatchSize": 1, + "batchSize": None, + "scheduling": "dynamic", + "offload": {"ditCpu": False}, + }, + "hardwareHealth": { + "status": "unknown", + "reason": "Missing clocks", + "evidence": None, + }, + "quality": { + "scale": "ordinal_0_to_4", + "contractId": None, + "contractSha256": None, + "rubricVersion": "fixture-v0", + "rubricSha256": None, + "metrics": { + "prompt_adherence": { + "status": "unjudged", + "direction": "higher", + "value": None, + "samples": 0, + "total": 20, + "calibration": None, + } + }, + }, + } + }, + } + + +def test_nullable_unjudged_projection_and_explicit_hardware_failure_are_valid_shapes(): + record = sidecar() + VALIDATOR.validate(record) + failed = deepcopy(record) + failed["cells"]["c1"]["hardwareHealth"] = { + "status": "fail", + "reason": "Thermal throttle", + "evidence": "health.json", + } + VALIDATOR.validate(failed) + minimal = deepcopy(record) + minimal["cells"]["c1"] = {"runSha256": "b" * 64, "specSha256": "c" * 64} + VALIDATOR.validate(minimal) + + +@pytest.mark.parametrize( + "defect", + [ + "unknown_root", + "unknown_cell", + "hash", + "empty_cells", + "batch_zero", + "batch_fraction", + "offload_string", + "unknown_health", + "universal_quality", + "score_range", + "legacy_scale", + "missing_scale", + "legacy_dimension", + "fractional_samples", + "calibration_date", + ], +) +def test_malformed_or_invented_projection_fields_are_rejected(defect): + record = sidecar() + cell = record["cells"]["c1"] + metric = cell["quality"]["metrics"]["prompt_adherence"] + if defect == "unknown_root": + record["releaseQualified"] = True + elif defect == "unknown_cell": + cell["universalQuality"] = 100 + elif defect == "hash": + cell["runSha256"] = "not-a-source-checksum" + elif defect == "empty_cells": + record["cells"] = {} + elif defect == "batch_zero": + cell["deployment"]["maxBatchSize"] = 0 + elif defect == "batch_fraction": + cell["deployment"]["batchSize"] = 1.5 + elif defect == "offload_string": + cell["deployment"]["offload"]["ditCpu"] = "false" + elif defect == "unknown_health": + cell["hardwareHealth"]["status"] = "probably-good" + elif defect == "universal_quality": + cell["quality"]["metrics"]["overall"] = metric + elif defect == "score_range": + metric["value"] = 6 + elif defect == "legacy_scale": + cell["quality"]["scale"] = "ordinal_1_to_5" + elif defect == "missing_scale": + del cell["quality"]["scale"] + elif defect == "legacy_dimension": + cell["quality"]["metrics"] = {"subject_consistency": metric} + elif defect == "fractional_samples": + metric["samples"] = 1.5 + else: + metric["calibration"] = {"status": "calibrated", "frozenAt": "not-a-date"} + with pytest.raises(jsonschema.ValidationError): + VALIDATOR.validate(record) + + +def test_original_seven_dimension_projection_keeps_zero_scores(): + record = sidecar() + quality = record["cells"]["c1"]["quality"] + measurement = quality["metrics"]["prompt_adherence"] + measurement.update(value=0, status="judged_unqualified", samples=20) + quality["metrics"] = { + name: deepcopy(measurement) + for name in ( + "prompt_adherence", + "visual_fidelity", + "temporal_consistency", + "motion_plausibility", + "audio_quality", + "audio_content", + "av_sync", + ) + } + VALIDATOR.validate(record) diff --git a/experimental/video-generation/tests/test_mvp_deployment_frontier.py b/experimental/video-generation/tests/test_mvp_deployment_frontier.py new file mode 100644 index 0000000000..63a5914821 --- /dev/null +++ b/experimental/video-generation/tests/test_mvp_deployment_frontier.py @@ -0,0 +1,294 @@ +"""Synthetic frontend/backend parity; these scores are not measured evidence.""" + +import json +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +from evaluator.mvp_deployment_frontier import pareto_frontier, select_frontiers + +FIXTURES = json.loads( + (Path(__file__).parent / "fixtures/deployment-frontier.json").read_text() +) + + +@pytest.mark.parametrize("case", FIXTURES["cases"], ids=lambda case: case["name"]) +def test_shared_dominance_cases(case): + result = pareto_frontier(case["points"], case["xBetter"], case["yBetter"]) + assert [point["id"] for point in result] == case["expected"] + + +def test_nonfinite_missing_and_boolean_axes_do_not_become_measurements(): + invalid = [None, float("nan"), float("inf"), "12", True] + points = [{"id": "measured", "x": 10, "y": 5}] + points += [ + {"id": str(index), "x": value, "y": 100} for index, value in enumerate(invalid) + ] + assert pareto_frontier(points, "lower", "higher") == [ + {"id": "measured", "x": 10, "y": 5} + ] + assert pareto_frontier([], "lower", "higher") == [] + + +def record(identity, **overrides): + return { + "id": identity, + "hardwareKey": "h200", + "cohortKey": "matched-workload", + "deploymentKey": identity, + "queueingStatus": "unqueued", + "x": 150, + "y": 6, + **overrides, + } + + +def test_cohorts_health_queue_and_quality_filter_before_dominance(): + four = assessed_record("four-gpu") + other_evaluator = assessed_record("other-evaluator", x=2, y=100) + other_evaluator["quality"]["metrics"]["audio_content"]["evaluatorVersion"] = "v2" + points = [ + four, + assessed_record("eight-gpu", x=80, y=5), + assessed_record("dominated", x=160, y=5), + assessed_record("other-workload", x=1, y=100, cohortKey="other"), + other_evaluator, + assessed_record( + "failed-hardware", x=1, y=1000, hardwareHealth={"status": "fail"} + ), + assessed_record("queue", x=1, y=1000, queueingStatus="queueing"), + assessed_record("unknown-capacity", x=1, y=1000, queueingStatus="unknown"), + record("unjudged", x=1, y=1000), + assessed_record("unknown-workload-a", cohortKey=None), + assessed_record("unknown-workload-b", cohortKey=None, x=1, y=100), + ] + result = select_frontiers(points, "lower", "higher", quality_required=True) + assert [point["id"] for point in result["plotted"]] == [ + "four-gpu", + "eight-gpu", + "other-workload", + "other-evaluator", + "unknown-workload-a", + "unknown-workload-b", + ] + assert [ + [point["id"] for point in line] for line in result["frontiers"].values() + ] == [ + ["eight-gpu", "four-gpu"], + ["other-workload"], + ["other-evaluator"], + ["unknown-workload-a"], + ["unknown-workload-b"], + ] + assert result["excluded"] == { + "failed-hardware": "hardware_health_failed", + "queue": "queueing", + "unknown-capacity": "unknown_capacity", + "unjudged": "quality_unqualified", + } + result = select_frontiers( + points, "lower", "higher", quality_required=True, optimal=False + ) + assert [point["id"] for point in result["plotted"]][:3] == [ + "four-gpu", + "eight-gpu", + "dominated", + ] + assert not result["plotted"][2]["optimal"] + assert len(result["plotted"]) == 7 + + +def test_repeat_observations_of_one_deployment_are_unconnected(): + result = select_frontiers( + [ + record("c1", deploymentKey="same"), + record("c2", deploymentKey="same", x=160, y=7), + ], + "lower", + "higher", + ) + assert len(result["plotted"]) == 2 + assert all(len(line) == 1 for line in result["frontiers"].values()) + assert result["multiLayout"] is False + + +def test_offline_cli_writes_selected_identities_without_rewriting_inputs(tmp_path): + source = tmp_path / "records.json" + source.write_text(json.dumps({"points": [record("fast", x=80), record("slow")]})) + original = source.read_bytes() + output = tmp_path / "frontier.json" + subprocess.run( + [ + sys.executable, + "-m", + "evaluator.mvp_deployment_frontier", + str(source), + "--output", + str(output), + "--x-better", + "lower", + "--y-better", + "higher", + ], + check=True, + cwd=Path(__file__).resolve().parents[1], + ) + result = json.loads(output.read_text()) + assert [point["id"] for point in result["plotted"]] == ["fast"] + assert source.read_bytes() == original + + +def assessed_record(identity="qualified", **overrides): + metric = { + "value": 4, + "status": "pass", + "direction": "higher", + "evaluatorId": "fixture-human", + "evaluatorVersion": "v1", + "evaluatorSha256": "a" * 64, + "samples": 20, + "total": 20, + "calibration": { + "status": "calibrated", + "cohortId": "pilot-v1", + "threshold": 3, + "provenance": "fixture://frozen-rule", + "frozenAt": "2026-09-01T00:00:00Z", + }, + } + return record( + identity, + samples=20, + quality={ + "scale": "ordinal_0_to_4", + "contractId": "fixture-contract", + "contractSha256": "b" * 64, + "rubricVersion": "rubric-v1", + "rubricSha256": "c" * 64, + "metrics": { + name: deepcopy(metric) + for name in ( + "prompt_adherence", + "visual_fidelity", + "temporal_consistency", + "motion_plausibility", + "audio_quality", + "audio_content", + "av_sync", + ) + }, + }, + **overrides, + ) + + +def test_complete_original_rubric_accepts_a_calibrated_zero_without_coercion(): + point = assessed_record() + metric = point["quality"]["metrics"]["prompt_adherence"] + metric["value"] = metric["calibration"]["threshold"] = 0 + result = select_frontiers([point], "lower", "higher", quality_required=True) + assert [p["id"] for p in result["plotted"]] == ["qualified"] + assert result["plotted"][0]["quality"]["metrics"]["prompt_adherence"]["value"] == 0 + + +@pytest.mark.parametrize( + "defect", + [ + "legacy_scale", + "missing_scale", + "missing_dimension", + "failed_dimension", + "coverage", + "uncalibrated", + ], +) +def test_opaque_claim_cannot_admit_incomplete_or_rejected_quality(defect): + point = assessed_record(qualifiedQualityCohort="producer-claimed-pass") + quality = point["quality"] + if defect == "legacy_scale": + quality["scale"] = "ordinal_1_to_5" + elif defect == "missing_scale": + del quality["scale"] + elif defect == "missing_dimension": + del quality["metrics"]["audio_content"] + elif defect == "failed_dimension": + quality["metrics"]["audio_content"]["status"] = "fail" + elif defect == "coverage": + quality["metrics"]["audio_content"]["samples"] = 19 + else: + quality["metrics"]["audio_content"]["calibration"]["status"] = "uncalibrated" + result = select_frontiers([point], "lower", "higher", quality_required=True) + assert result["plotted"] == [] + assert result["excluded"] == {"qualified": "quality_unqualified"} + + +def test_reader_threshold_only_tightens_the_selected_dimension(): + point = assessed_record() + audio = point["quality"]["metrics"]["audio_content"] + audio["value"] = audio["calibration"]["threshold"] = 1 + result = select_frontiers( + [point], + "lower", + "higher", + quality_required=True, + quality_metric="prompt_adherence", + quality_threshold=4, + ) + assert [p["id"] for p in result["plotted"]] == ["qualified"] + result = select_frontiers( + [point], + "lower", + "higher", + quality_required=True, + quality_metric="audio_content", + quality_threshold=2, + ) + assert result["plotted"] == [] + audio["value"] = 0 + result = select_frontiers( + [point], + "lower", + "higher", + quality_required=True, + quality_metric="audio_content", + quality_threshold=0, + ) + assert result["plotted"] == [] + + +@pytest.mark.parametrize( + "field,value", + [ + ("evaluatorVersion", "v2"), + ("calibration.threshold", 2), + ("calibration.provenance", "fixture://another-rule"), + ], +) +def test_unselected_dimensions_keep_incompatible_quality_cohorts_separate(field, value): + first = assessed_record("original") + second = assessed_record("other-rule", x=10, y=100) + metric = second["quality"]["metrics"]["av_sync"] + if field.startswith("calibration."): + metric["calibration"][field.split(".")[1]] = value + else: + metric[field] = value + result = select_frontiers([first, second], "lower", "higher", quality_required=True) + assert [p["id"] for p in result["plotted"]] == ["original", "other-rule"] + assert [len(line) for line in result["frontiers"].values()] == [1, 1] + assert ( + result["plotted"][0]["qualifiedQualityCohort"] + != result["plotted"][1]["qualifiedQualityCohort"] + ) + + +def test_opaque_quality_claim_alone_never_qualifies(): + result = select_frontiers( + [record("claim", qualifiedQualityCohort="pass")], + "lower", + "higher", + quality_required=True, + ) + assert result["excluded"] == {"claim": "quality_unqualified"} diff --git a/experimental/video-generation/tests/test_srt_selected.py b/experimental/video-generation/tests/test_srt_selected.py index b2ff328a87..193336e155 100644 --- a/experimental/video-generation/tests/test_srt_selected.py +++ b/experimental/video-generation/tests/test_srt_selected.py @@ -52,22 +52,54 @@ def test_h3_vllm_command_uses_the_tested_two_gpu_fl2va_profile(): mvp_srt_selected.server_argv(plan, "vllm-omni", Path("/model"), 30283, 4) +def test_wan_selected_plan_uses_the_same_video_request_on_both_engines(): + path = PACKAGE / "specs/wan.yaml" + spec = h3_spec.load_spec(path) + expected = h3_spec.plan_sha256(h3_spec.plan_for(spec, "release-v1")) + plan = mvp_srt_selected.selected_plan(path, "release-v1", expected) + sglang = preview_plan(plan, runtime="sglang") + vllm = preview_plan(plan, runtime="vllm-omni") + assert sglang["plan_sha256"] == vllm["plan_sha256"] == expected + assert sglang["measurement_count"] == vllm["measurement_count"] == 20 + assert sglang["total_requests"] == vllm["total_requests"] == 21 + assert sglang["slots"][1]["request"] == vllm["slots"][1]["request"] + + +def test_wan_commands_use_the_recorded_eight_gpu_profiles(tmp_path): + plan = h3_spec.plan_for(h3_spec.load_spec(PACKAGE / "specs/wan.yaml"), "release-v1") + sglang = mvp_srt_selected.server_argv( + plan, "sglang", Path("/model"), 30283, 8, output=tmp_path + ) + assert sglang[sglang.index("--num-gpus") + 1] == "8" + assert sglang[sglang.index("--ulysses-degree") + 1] == "8" + assert sglang[sglang.index("--tp-size") + 1] == "1" + assert sglang[sglang.index("--output-path") + 1] == str(tmp_path / "generated") + vllm = mvp_srt_selected.server_argv(plan, "vllm-omni", Path("/model"), 30283, 8) + assert vllm[:3] == ["vllm", "serve", "/model"] + assert vllm[vllm.index("--cfg-parallel-size") + 1] == "2" + assert vllm[vllm.index("--usp") + 1] == "4" + assert vllm[vllm.index("--stage-init-timeout") + 1] == "3600" + with pytest.raises(ValueError, match="eight Wan GPUs"): + mvp_srt_selected.server_argv(plan, "vllm-omni", Path("/model"), 30283, 4) @pytest.mark.parametrize("runtime", ["sglang", "vllm-omni"]) -def test_selected_server_seals_its_bound_execution_spec(tmp_path, monkeypatch, runtime): - plan = h3_spec.plan_for(h3_spec.load_spec(PACKAGE / "specs/h3.yaml"), "release-v1") +@pytest.mark.parametrize("model_name,gpus", [("h3", 2), ("wan", 8)]) +def test_selected_server_seals_its_bound_execution_spec(tmp_path, monkeypatch, runtime, model_name, gpus): + plan = h3_spec.plan_for(h3_spec.load_spec(PACKAGE / f"specs/{model_name}.yaml"), "release-v1") model = tmp_path / "model" - (model / "FL2VA").mkdir(parents=True) - (model / "Ref2VA").mkdir() - for name in ("model_index.json", "LICENSE", "README.md", "Ref2VA/model_index.json"): - (model / name).write_text(name) - uuids = ["GPU-1", "GPU-2"] + model.mkdir() + if model_name == "h3": + (model / "FL2VA").mkdir() + (model / "Ref2VA").mkdir() + for name in ("model_index.json", "LICENSE", "README.md", "Ref2VA/model_index.json"): + (model / name).write_text(name) + uuids = [f"GPU-{index}" for index in range(gpus)] monkeypatch.setattr(mvp_gpu_job, "cuda_devices", lambda: uuids) executed = [] monkeypatch.setattr(mvp_srt_selected.os, "execvpe", lambda file, argv, env: executed.append((file, argv))) output = tmp_path / "server" mvp_srt_selected.serve( plan, runtime=runtime, runtime_revision="a" * 40, - model_path=model, port=30283, gpus=2, output=output, + model_path=model, port=30283, gpus=gpus, output=output, ) spec = json.loads((output / "spec.json").read_text()) launch = json.loads((output / "launch.json").read_text()) @@ -76,7 +108,7 @@ def test_selected_server_seals_its_bound_execution_spec(tmp_path, monkeypatch, r assert spec["baseline"]["name"] == launch["runtime"] == runtime assert spec["baseline"]["revision"] == launch["runtime_revision"] == "a" * 40 assert spec["baseline"]["argv"] == launch["argv"] == executed[0][1] - if runtime == "vllm-omni": + if runtime == "vllm-omni" and model_name == "h3": assert spec["baseline"]["model_path"] == executed[0][1][2] else: assert spec["baseline"]["model_path"] == str(model) diff --git a/experimental/video-generation/tests/test_wan_ci.py b/experimental/video-generation/tests/test_wan_ci.py new file mode 100644 index 0000000000..8abdbb0074 --- /dev/null +++ b/experimental/video-generation/tests/test_wan_ci.py @@ -0,0 +1,100 @@ +import sys + +import pytest + +import ci +from test_ci import config +from test_wan_runner import wan_plan # noqa: F401 + + +def prepared_config(tmp_path, plan): + cfg = config(tmp_path) + cfg.update(mode="serving-smoke", concurrencies=[1]) + ci.write(cfg["spec"]["path"], {"plan": plan}) + cfg["spec"]["sha256"] = ci.digest(cfg["spec"]["path"]) + return cfg + + +def test_wan_routing_requires_a_sealed_serving_plan(wan_plan, tmp_path): + cfg = prepared_config(tmp_path, wan_plan) + assert ci.prepared_artifact_prefix(cfg) == "video-serving" + cfg["mode"] = "smoke" + with pytest.raises(ValueError, match="serving-smoke"): + ci.prepared_artifact_prefix(cfg) + cfg["mode"] = "serving-smoke" + ci.write(cfg["spec"]["path"], {"plan": {**wan_plan, "repetitions": 2}}) + with pytest.raises(ValueError, match="changed"): + ci.prepared_artifact_prefix(cfg) + + +def test_preparation_failure_keeps_model_and_planned_counts_without_gpu_identity(wan_plan, tmp_path, monkeypatch): + cfg = prepared_config(tmp_path, wan_plan) + path, output = tmp_path / "site.json", tmp_path / "output" + ci.write(path, cfg) + for name, value in {"GITHUB_RUN_ID": "10001005", "GITHUB_RUN_ATTEMPT": "1", + "H3_SOURCE_SHA": "a" * 40, "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX"}.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr(sys, "argv", ["ci.py", "--config", str(path), "--output", str(output)]) + monkeypatch.setattr(ci, "command", lambda argv, **kw: "a" * 40 if "rev-parse" in argv else "") + monkeypatch.setattr(ci, "allocate", lambda *args, **kw: pytest.fail("preparation failure must not allocate")) + monkeypatch.setattr(ci, "recover", lambda *args, **kw: pytest.fail("preparation failure must not inspect GPUs")) + assert ci.main() == 2 + manifest = ci.read(output / "manifest.json") + assert manifest["workload_plan"] == wan_plan + assert manifest["slurm_allocation"] is None + assert manifest["execution_started"] is False + matrix = ci.read(output / "serving-smoke.json") + assert matrix["bundle_type"] == "video_serving_smoke_matrix" + assert matrix["completion"] == {"scheduled": 4, "attempted": 0, "completed": 0, + "valid": 0, "failed": 4, "not_started": 4, "unfinished": 0} + assert "gpu_uuids" not in matrix + assert "runtime" not in matrix + assert "run" not in matrix["cells"][0] + expected = {name: digest for digest, name in (line.split(" ") for line in (output / "SHA256SUMS").read_text().splitlines())} + assert ci.inventory(output) == expected + + +def test_failure_receipt_never_rewrites_existing_execution(wan_plan, tmp_path): + cfg = prepared_config(tmp_path, wan_plan) + output = tmp_path / "artifact" + output.mkdir() + ci.write(output / "manifest.json", {"existing": "preserve"}) + ci.record_wan_preparation_failure(cfg, output, "later failure") + assert ci.read(output / "manifest.json") == {"existing": "preserve"} + assert not (output / "serving-smoke.json").exists() + + +@pytest.mark.parametrize("collection_failed", [False, True]) +def test_preparation_fallback_preserves_a_persistent_run(wan_plan, tmp_path, monkeypatch, collection_failed): + cfg = prepared_config(tmp_path, wan_plan) + for name, value in {"GITHUB_RUN_ID": "10001005", "GITHUB_RUN_ATTEMPT": "1", + "H3_SOURCE_SHA": "a" * 40, "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX"}.items(): + monkeypatch.setenv(name, value) + run_dir = tmp_path / "results" / cfg["task_id"] / "github-10001005-1" + run_dir.mkdir(parents=True) + ci.write(run_dir / "ci.json", {"phase": "failed", "execution_started": True}) + ci.write(run_dir / "manifest.json", {"slurm_allocation": {"identity": {"JobId": "123"}}}) + ci.write(run_dir / "serving-smoke.json", {"completion": {"attempted": 2, "not_started": 2}}) + output = tmp_path / "artifact" + output.mkdir() + if collection_failed: + copyfile = ci.shutil.copyfile + + def interrupted_copy(source, target): + if source.name == "manifest.json": + raise OSError("collection interrupted before manifest copy") + return copyfile(source, target) + + monkeypatch.setattr(ci.shutil, "copyfile", interrupted_copy) + with pytest.raises(OSError, match="collection interrupted"): + ci.collect(run_dir, output) + + ci.record_wan_preparation_failure(cfg, output, "collection failure or existing-run retry") + + assert ci.read(run_dir / "serving-smoke.json")["completion"] == {"attempted": 2, "not_started": 2} + assert not (output / "manifest.json").exists() + assert not (output / "serving-smoke.json").exists() + if collection_failed: + assert ci.read(output / "ci.json") == {"phase": "failed", "execution_started": True} + else: + assert list(output.iterdir()) == [] diff --git a/experimental/video-generation/tests/test_wan_gpu_job.py b/experimental/video-generation/tests/test_wan_gpu_job.py new file mode 100644 index 0000000000..b0f537a9bb --- /dev/null +++ b/experimental/video-generation/tests/test_wan_gpu_job.py @@ -0,0 +1,160 @@ +"""CPU-only Wan launch and evidence checks; fixture bytes are not model output.""" + +import time + +import pytest + +from evaluator import mvp_gpu_evidence as evidence +from evaluator import mvp_gpu_job as gpu +from evaluator import cli +from test_mvp_gpu_job import spec # noqa: F401 +from test_mvp_serving_smoke import saved_single + + +@pytest.fixture +def wan_spec(spec): + spec["plan"].update( + model_id="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + generation={ + "width": 832, "height": 480, "frame_count": 81, "fps": 16, + "num_inference_steps": 40, "guidance_scale": 4.0, + "guidance_scale_2": 3.0, "flow_shift": 12.0, "negative_prompt": "", + }, + ) + spec["plan"]["cases"] = [{ + "case_id": "walking", "prompt": "A person walks across a park.", "seed": 11, + "requires_motion": True, "requires_sound": False, + }] + spec["plan"]["repetitions"] = 4 + spec["serving"] = {"concurrency": 1} + spec["policy"] = {"policy_id": "wan-unqualified-smoke", "calibration_status": "uncalibrated"} + spec["server"] = { + "ulysses_degree": 1, "tp_size": 1, "dit_cpu_offload": False, "dit_layerwise_offload": False, + "text_encoder_cpu_offload": True, "vae_cpu_offload": False, + } + return spec + + +def test_wan_launch_uses_video_model_and_explicit_offload_controls(wan_spec, tmp_path): + output = str(tmp_path / "job/supervisor/baseline/generated") + command = gpu._server_argv(wan_spec, "baseline", output_path=output) + assert command[3:] == [ + "serve", "--model-type", "diffusion", "--model-path", wan_spec["model"]["path"], + "--model-id", "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "--revision", wan_spec["model"]["revision"], "--num-gpus", "1", + "--ulysses-degree", "1", "--tp-size", "1", "--ring-degree", "1", + "--enable-cfg-parallel", "false", "--host", "127.0.0.1", + "--port", "30280", "--enable-torch-compile", "false", "--dit-cpu-offload", "false", + "--dit-layerwise-offload", "false", + "--text-encoder-cpu-offload", "true", "--vae-cpu-offload", "false", + "--warmup-mode", "off", "--output-path", output, + ] + + +def test_wan_preview_is_single_runtime_without_qualification(wan_spec): + result = gpu.preview_gpu_job(wan_spec) + assert set(result["commands"]) == {"baseline"} + assert result["sequence"] == [ + "verify pinned files", "acquire UUID locks", "verify idle", + "baseline startup/warmup/measure/cleanup", + ] + assert result["workload"]["total_requests"] == 5 + assert result["evidence_kind"] == "gpu_job_preview_no_execution" + + +@pytest.mark.parametrize("field,value", [ + ("calibration_status", "operator_calibrated"), ("policy_id", " "), + ("min_audio_spectral_cosine", 0.95), +]) +def test_wan_does_not_accept_a_paired_quality_policy(wan_spec, field, value): + wan_spec["policy"][field] = value + with pytest.raises(ValueError, match="Wan.*policy"): + gpu.validate_gpu_job(wan_spec) + + +@pytest.mark.parametrize("field,value", [ + ("encoder_parallel", "auto"), ("performance_mode", "speed"), + ("layerwise_offload", {"components": ["dit"], "prefetch_size": 1, "resident_layers": 20}), + ("attention_backend", "aiter"), ("vae_cpu_offload", "false"), ("dit_layerwise_offload", True), +]) +def test_wan_server_rejects_unsupported_or_implicit_controls(wan_spec, field, value): + wan_spec["server"][field] = value + with pytest.raises(ValueError, match="server"): + gpu.validate_gpu_job(wan_spec) + + +@pytest.mark.parametrize("mutation", ["no-serving", "amd", "h3-timing"]) +def test_wan_rejects_unqualified_runtime_paths(wan_spec, mutation): + if mutation == "no-serving": + wan_spec.pop("serving") + elif mutation == "amd": + wan_spec["gpu_vendor"] = "amd" + wan_spec["gpu_uuids"] = [value.removeprefix("GPU-") for value in wan_spec["gpu_uuids"]] + else: + wan_spec["server_timing"] = True + with pytest.raises(ValueError, match="Wan"): + gpu.validate_gpu_job(wan_spec) + + +@pytest.mark.parametrize("count,tp", [(3, 1), (2, 2)]) +def test_wan_rejects_unsupported_attention_layouts(wan_spec, count, tp): + wan_spec["gpu_uuids"] = [f"GPU-{index:08x}-abcd-abcd-abcd-123456789abc" for index in range(count)] + wan_spec["server"].update(ulysses_degree=count, tp_size=tp) + with pytest.raises(ValueError, match="Wan.*TP1"): + gpu.validate_gpu_job(wan_spec) + + +def test_wan_paired_execution_is_rejected_before_creating_evidence(wan_spec, tmp_path): + output = tmp_path / "paired" + with pytest.raises(ValueError, match="Wan.*serving"): + gpu.run_gpu_job(wan_spec, output) + assert not output.exists() + + +def test_wan_execution_retains_authorization_barrier(wan_spec, tmp_path): + wan_spec["authorization"]["compute_approved"] = False + output = tmp_path / "unapproved" + with pytest.raises(ValueError, match="approval"): + gpu.run_gpu_job(wan_spec, output, serving_smoke=True) + assert not output.exists() + + +def test_wan_serving_cli_retains_failed_receipt_without_gpu_work(wan_spec, tmp_path): + wan_spec["authorization"]["compute_approved"] = False + source = tmp_path / "spec.json" + gpu._write(source, wan_spec) + output = tmp_path / "serving" + + assert cli.main(["serving-smoke", "--spec", str(source), "--output", str(output), + "--concurrencies", "1"]) == 1 + result = gpu._read(output / "serving-smoke.json") + assert result["bundle_type"] == "video_serving_smoke_matrix" + assert result["status"] == "failed" + assert result["completion"]["scheduled"] == 4 + assert result["completion"]["attempted"] == 0 + assert not (output / "gpu").exists() + + +def test_wan_evidence_is_model_bound_and_cannot_qualify_paired_jobs(wan_spec, tmp_path): + directory = tmp_path / "saved-wan" + receipt = saved_single(wan_spec, directory) + receipt["evidence_kind"] = "controlled_video_gpu" + run_path = directory / "baseline/run.json" + run = gpu._read(run_path) + run["evidence_kind"] = "live_video" + gpu._write(run_path, run) + receipt["roles"]["baseline"]["run_sha256"] = gpu._hash(run_path) + gpu._write(directory / "gpu-job.json", receipt) + verified = evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5, serving_smoke=True) + assert set(verified["runs"]) == {"baseline"} + assert verified["comparison"] is None + assert verified["receipt"]["ci_accepted"] is False + assert verified["receipt"]["release_qualified"] is False + with pytest.raises(ValueError, match="Wan.*serving"): + evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5) + run["evidence_kind"] = "live_h3" + gpu._write(run_path, run) + receipt["roles"]["baseline"]["run_sha256"] = gpu._hash(run_path) + gpu._write(directory / "gpu-job.json", receipt) + with pytest.raises(ValueError, match="unsupported run evidence"): + evidence.verify_measurement_job(directory, deadline=time.monotonic() + 5, serving_smoke=True) diff --git a/experimental/video-generation/tests/test_wan_runner.py b/experimental/video-generation/tests/test_wan_runner.py new file mode 100644 index 0000000000..e33d5f94b6 --- /dev/null +++ b/experimental/video-generation/tests/test_wan_runner.py @@ -0,0 +1,187 @@ +"""CPU transport/media acceptance; encoded geometry is not Wan generation.""" + +import copy + +import pytest +import test_mvp_runner +from test_mvp_media import encode_media + +from evaluator import mvp_runner as runner + +fixture_server = test_mvp_runner.fixture_server + + +@pytest.fixture +def wan_plan(): + return { + "plan_id": "controlled-wan-request", + "model_id": runner.WAN_MODEL_ID, + "model_revision": "b" * 40, + "generation": { + "width": 64, + "height": 48, + "frame_count": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "", + }, + "cases": [ + { + "case_id": f"motion-{index}", + "prompt": "A person walks across a room.", + "seed": 11 + index, + "requires_motion": True, + "requires_sound": False, + } + for index in range(4) + ], + "repetitions": 1, + "warmup_runs": 1, + } + + +@pytest.mark.parametrize("runtime", ["sglang", "vllm-omni"]) +def test_wan_preview_preserves_explicit_request_controls(wan_plan, runtime): + preview = runner.preview_plan(wan_plan, runtime=runtime) + assert preview["measurement_count"] == 4 + assert preview["warmup_count"] == 1 + assert preview["slots"][0]["request"] == { + "model": runner.WAN_MODEL_ID, + "prompt": "A person walks across a room.", + "seed": 11, + "width": 64, + "height": 48, + "num_frames": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "", + } + + +@pytest.mark.parametrize( + "change", + [ + {"width": 833}, + {"frame_count": 80}, + {"guidance_scale_2": None}, + {"audio_flow_shift": 3}, + {"duration_seconds": 5}, + {"negative_prompt": None}, + {"guidance_scale": 1}, + {"flow_shift": 5}, + ], +) +def test_wan_rejects_ignored_or_rounded_controls(wan_plan, change): + wan_plan["generation"].update(change) + with pytest.raises(ValueError): + runner.preview_plan(wan_plan) + + +def test_wan_rejects_audio_requirement(wan_plan): + wan_plan["cases"][0]["requires_sound"] = True + with pytest.raises(ValueError, match="video-only"): + runner.preview_plan(wan_plan) + + +def test_unknown_runtime_fails_before_creating_a_run(wan_plan, tmp_path): + with pytest.raises(ValueError, match="sglang or vllm-omni"): + runner.preview_plan(wan_plan, runtime="other") + output = tmp_path / "run" + with pytest.raises(ValueError, match="sglang or vllm-omni"): + runner.run_plan( + wan_plan, + output, + endpoint="http://127.0.0.1:1", + runtime="other", + runtime_revision="a" * 40, + hardware_label="CPU fixture", + model_revision=wan_plan["model_revision"], + ) + assert not output.exists() + + +@pytest.mark.parametrize("runtime", ["sglang", "vllm-omni"]) +@pytest.mark.parametrize( + "mode,audio_mode,valid", + [("success", "absent", 1), ("success", "stereo", 0), ("job_failed", "absent", 0)], +) +def test_wan_http_delivery_uses_real_video_only_decode( + wan_plan, tmp_path, monkeypatch, fixture_server, runtime, mode, audio_mode, valid +): + plan = copy.deepcopy(wan_plan) + plan["cases"] = plan["cases"][:1] + plan["warmup_runs"] = 0 + media = encode_media( + tmp_path / "cpu-fixture.mp4", + width=64, + height=48, + fps=16, + frame_count=81, + audio_mode=audio_mode, + ) + monkeypatch.setattr(test_mvp_runner, "FIXTURE_BYTES", media.read_bytes()) + endpoint, state = fixture_server(mode=mode) + result = runner.run_plan( + plan, + tmp_path / "run", + endpoint=endpoint, + runtime=runtime, + runtime_revision="a" * 40, + hardware_label="CPU HTTP fixture; no GPU", + model_revision=plan["model_revision"], + timeout_seconds=10, + serving_concurrency=1, + ) + assert result["summary"]["scheduled"] == 1 + assert result["summary"]["valid"] == valid + assert result["configuration"]["model_id"] == runner.WAN_MODEL_ID + assert result["configuration"]["capabilities"]["audio_generation"] == "unsupported" + assert result["configuration"]["protocol_source"]["repository"] == ( + "https://github.com/sgl-project/sglang" + if runtime == "sglang" + else "https://github.com/vllm-project/vllm-omni" + ) + assert state["requests"][:2] == [ + ("POST", "/v1/videos"), + ("GET", "/v1/videos/fixture-1"), + ] + expected_post = { + "model": runner.WAN_MODEL_ID, + "prompt": "A person walks across a room.", + "seed": 11, + "width": 64, + "height": 48, + "num_frames": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "", + } + if runtime == "vllm-omni": + expected_post = {key: str(value) for key, value in expected_post.items()} + assert state["posts"] == [expected_post] + record = result["records"][0] + assert record["expected_media"]["audio_required"] is False + assert "audio_sample_rate_hz" not in record["expected_media"] + if valid: + assert state["requests"][-1] == ("GET", "/v1/videos/fixture-1/content") + assert record["media"]["valid"] is True + assert record["media"]["audio"]["present"] is False + assert record["sha256"] + elif mode == "job_failed": + assert record["artifact_path"] is None + assert all(not path.endswith("/content") for _, path in state["requests"]) + else: + assert record["outcome"] == "invalid_media" + assert any( + check["name"] == "audio.absent" and check["status"] == "failed" + for check in record["media"]["checks"] + ) diff --git a/experimental/video-generation/tests/test_wan_spec.py b/experimental/video-generation/tests/test_wan_spec.py new file mode 100644 index 0000000000..a2ebb84752 --- /dev/null +++ b/experimental/video-generation/tests/test_wan_spec.py @@ -0,0 +1,127 @@ +"""Load a controlled Wan spec into the runner's observable plan contract.""" + +import copy + +import h3_spec +import pytest +import yaml + +from evaluator import mvp_runner + + +def controlled_spec(): + citation = "example@abcdef1:spec.yaml:1" + values = { + "width": 64, + "height": 48, + "frame_count": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "", + } + return { + "schema_version": 1, + "model": { + "id": {"value": mvp_runner.WAN_MODEL_ID, "source": citation}, + "revision": {"value": "b" * 40, "source": citation}, + }, + "generation": { + name: { + "value": value, + "source": citation, + "sent": {"sglang": True, "vllm-omni": True}, + "checked_on_clip": name in {"width", "height", "frame_count", "fps"}, + } + for name, value in values.items() + }, + "prompts": { + "walk": { + "text": "A person walks across a room.", + "requires_motion": True, + "requires_sound": False, + "source": citation, + }, + }, + "sets": { + "sample": { + "plan_id": "controlled-wan", + "repetitions": 2, + "warmup_runs": 1, + "seed_sources": {7: citation}, + "cases": [{"case_id": "walk-s7", "prompt": "walk", "seed": 7}], + }, + }, + "reference": {"note": "ignored by the plan hash"}, + } + + +def write_spec(path, spec): + path.write_text(yaml.safe_dump(spec), encoding="utf-8") + return h3_spec.load_spec(path) + + +def test_loader_builds_runner_plan_and_keeps_reference_out_of_hash(tmp_path): + source = controlled_spec() + loaded = write_spec(tmp_path / "spec.yaml", source) + plan = h3_spec.plan_for(loaded, "sample") + assert plan == { + "model_id": mvp_runner.WAN_MODEL_ID, + "model_revision": "b" * 40, + "generation": { + "width": 64, + "height": 48, + "frame_count": 81, + "fps": 16, + "num_inference_steps": 40, + "guidance_scale": 4.0, + "guidance_scale_2": 3.0, + "flow_shift": 12.0, + "negative_prompt": "", + }, + "plan_id": "controlled-wan", + "cases": [ + { + "case_id": "walk-s7", + "prompt": "A person walks across a room.", + "seed": 7, + "requires_motion": True, + "requires_sound": False, + } + ], + "repetitions": 2, + "warmup_runs": 1, + } + preview = mvp_runner.preview_plan(plan, runtime="vllm-omni") + assert preview["measurement_count"] == 2 + assert preview["warmup_count"] == 1 + assert preview["slots"][0]["request"]["num_frames"] == 81 + assert preview["plan_sha256"] == h3_spec.plan_sha256(plan) + changed = copy.deepcopy(source) + changed["reference"]["note"] = "another note" + changed["generation"]["fps"]["source"] = "another@abcdef1:spec.yaml:2" + assert h3_spec.spec_sha256( + write_spec(tmp_path / "other.yaml", changed) + ) == h3_spec.spec_sha256(loaded) + + +@pytest.mark.parametrize( + "mutation,match", + [ + ( + lambda spec: spec["sets"]["sample"]["cases"][0].update(seed=8), + "seed_sources", + ), + ( + lambda spec: spec["generation"]["fps"].update(source="uncited"), + "generation.fps.source", + ), + ], +) +def test_loader_rejects_uncited_or_unmapped_inputs(tmp_path, mutation, match): + source = controlled_spec() + mutation(source) + with pytest.raises(ValueError, match=match): + write_spec(tmp_path / "invalid.yaml", source) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/sglang/h200-bf16/serving-client.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/sglang/h200-bf16/serving-client.yaml new file mode 100644 index 0000000000..dbcfdc06bc --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/sglang/h200-bf16/serving-client.yaml @@ -0,0 +1,45 @@ +# Wan2.2 video serving on one H200 node through srt-slurm's services-only shape. +# +# The GPU-owning diffusion server is a generic service with an HTTP readiness +# probe; benchmark.type custom runs the H3 client against it. The video binder +# (infx.srt_slurm.video) writes model.*, resources.gpus_per_node, the native +# telemetry block, benchmark.concurrencies and every H3_* input the launcher +# owns (spec, package, model and result paths), so this fragment holds only the +# recipe-owned service and client shape. +schema: 2 +name: wan-videogen-serving-client + +resources: + gpu_type: h200 + +frontend: + type: none + +services: + - name: h3-server + type: generic + nodes: 1 + critical: true + start: before_workers + command: + - bash + - /infmax-workspace/benchmarks/h3/run_server.sh + env: + PYTHONUNBUFFERED: "1" + H3_SERVER_ROLE: baseline + H3_SERVER_PORT: "30283" + VIDEO_RUNTIME_REVISION: 71de97b264b04dcd514cf904003028aefe9775c8 + readiness: + http: + port: 30283 + path: /health + timeout_seconds: 2400 + interval_seconds: 5 + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/h3/run_client.sh + env: + PYTHONUNBUFFERED: "1" + H3_SERVER_SERVICE: h3-server + H3_SERVER_PORT: "30283" diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/vllm-omni/h200-bf16/serving-client.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/vllm-omni/h200-bf16/serving-client.yaml new file mode 100644 index 0000000000..38368911de --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/wan/vllm-omni/h200-bf16/serving-client.yaml @@ -0,0 +1,37 @@ +schema: 2 +name: wan-vllm-omni-serving-client + +resources: + gpu_type: h200 + +frontend: + type: none + +services: + - name: h3-server + type: generic + nodes: 1 + critical: true + start: before_workers + command: + - bash + - /infmax-workspace/benchmarks/h3/run_server.sh + env: + PYTHONUNBUFFERED: "1" + H3_SERVER_ROLE: baseline + H3_SERVER_PORT: "30283" + VIDEO_RUNTIME_REVISION: a8576ccb725c4e21cd13c3eb5f9a546b21149d2b + readiness: + http: + port: 30283 + path: /health + timeout_seconds: 2400 + interval_seconds: 5 + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/h3/run_client.sh + env: + PYTHONUNBUFFERED: "1" + H3_SERVER_SERVICE: h3-server + H3_SERVER_PORT: "30283" diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 6061b7968d..147ffbb7f0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9852,6 +9852,42 @@ h3-bf16-h200-vllm-omni-video: srt-recipe: serving-client.yaml conc-list: [1] +wan22-bf16-h200-sglang-video: + image: hongccc/sglang-omni:h3-runtime-71de97b + model: Wan-AI/Wan2.2-T2V-A14B-Diffusers + model-prefix: wan22 + runner: cluster:h200-dgxc + precision: bf16 + framework: sglang + multinode: false + srt-recipe-dir: wan/sglang/h200-bf16 + scenarios: + video-generation: + - spec: wan.yaml + set: release-v1 + search-space: + - gpus: 8 + srt-recipe: serving-client.yaml + conc-list: [1] + +wan22-bf16-h200-vllm-omni-video: + image: vllm/vllm-omni:v0.30.0@sha256:fc77cfaac7b1b43c30cbf32eaea214864434c6e0f303e6e9eb9f64104241a994 + model: Wan-AI/Wan2.2-T2V-A14B-Diffusers + model-prefix: wan22 + runner: cluster:h200-dgxc + precision: bf16 + framework: vllm-omni + multinode: false + srt-recipe-dir: wan/vllm-omni/h200-bf16 + scenarios: + video-generation: + - spec: wan.yaml + set: release-v1 + search-space: + - gpus: 8 + srt-recipe: serving-client.yaml + conc-list: [1] + h3-bf16-b200-sglang-video: # The B200 runtime is a CUDA 13 build whose SGLang lives in the /opt/sglang venv, so its # spec pins that interpreter; the workload, model and server settings match the H200 spec. diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index a748b31da6..fa6bd37174 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -111,6 +111,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| +| Wan2.2-T2V-A14B | `wan22` | 2026-10-18 | Video generation on H200 (SGLang and vLLM-Omni candidates; native qualification pending) | | | Muse Spark Open 816B-A42B | `musespark` | 2026-10-15 | Agentic coding (SGLang with DFlash2 draft `RadixArk/Muse-Spark-Open-DFlash2`; golden AL curve `musespark_dflash.yaml`) | | | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463, restored on srt-slurm 2026-09-30; GPU validation pending) | | | GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the deprecation policy) | Single-turn 1k1k (deprecated for all models before this model was added; never run) | diff --git a/inferencex-e2e/infx/tests/results/test_video_results.py b/inferencex-e2e/infx/tests/results/test_video_results.py index bc5ab94351..05122c05ac 100644 --- a/inferencex-e2e/infx/tests/results/test_video_results.py +++ b/inferencex-e2e/infx/tests/results/test_video_results.py @@ -4,8 +4,10 @@ from pathlib import Path import pytest +import yaml from infx.results import video +from infx.config import repository_root from infx.tests.results.power.test_aggregate_power_multinode import ( WINDOW_END, WINDOW_START, @@ -118,6 +120,23 @@ def test_service_planned_gpus_validate_with_clip_normalized_energy(tmp_path): assert summary["cells"]["4"]["status"] == "complete" and summary["status"] == "complete" +@pytest.mark.parametrize("runtime", ["sglang", "vllm-omni"]) +def test_wan_service_role_is_admitted_by_native_power(tmp_path, runtime): + recipe = repository_root() / f"benchmarks/single_node/srt-slurm-recipes/wan/{runtime}/h200-bf16/serving-client.yaml" + [service] = yaml.safe_load(recipe.read_text())["services"] + assert service["name"] == video.SERVICE + pkg = _service_package(tmp_path, service=service["name"]) + _cell(pkg, pkg.logs_root) + workspace = tmp_path / "ws" + workspace.mkdir() + assert video.run( + logs_root=pkg.logs_root, workspace=workspace, result_filename="wan_video", concurrencies=[4], + gpu_count=4, expected_producer_sha="a" * 40, identity=IDENTITY, require_power=True, + ) == 0 + sidecar = json.loads((workspace / "power_validation_wan_video_conc4.json").read_text()) + assert sidecar["power_valid"] and set(sidecar["per_gpu_role"].values()) == {service["name"]} + + def test_missing_power_never_publishes_as_measured_and_fails_only_when_required(tmp_path): pkg = _service_package(tmp_path) _cell(pkg, pkg.logs_root) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_video_selected.py b/inferencex-e2e/infx/tests/srt_slurm/test_video_selected.py index 95d3687379..25aeb00d79 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_video_selected.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_video_selected.py @@ -8,6 +8,7 @@ ROOT = repository_root() RECIPES = ROOT / "benchmarks/single_node/srt-slurm-recipes/h3" +WAN_RECIPES = ROOT / "benchmarks/single_node/srt-slurm-recipes/wan" ENV = { "IMAGE": "vllm/vllm-omni:v0.30.0", "MODEL": "MiniMaxAI/MiniMax-H3", @@ -56,6 +57,31 @@ def test_both_h3_engines_bind_the_same_selected_plan(): assert vllm["benchmark"]["env"]["H3_PLAN_SHA256"] == sglang["benchmark"]["env"]["H3_PLAN_SHA256"] +def test_both_wan_engines_bind_one_eight_gpu_plan(): + environment = { + **ENV, "MODEL": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "GPU_COUNT": "8", "H3_SPEC": "wan.yaml", + } + recipes = [ + video.prepare( + WAN_RECIPES / "vllm-omni/h200-bf16/serving-client.yaml", + environment, root=ROOT, + ), + video.prepare( + WAN_RECIPES / "sglang/h200-bf16/serving-client.yaml", + {**environment, "FRAMEWORK": "sglang", + "IMAGE": "hongccc/sglang-omni:h3-runtime-71de97b"}, root=ROOT, + ), + ] + assert {recipe["resources"]["gpus_per_node"] for recipe in recipes} == {8} + assert {recipe["benchmark"]["env"]["H3_PLAN_SHA256"] for recipe in recipes} == { + video.load_spec("wan.yaml", root=ROOT, set_id="release-v1")["plan_sha256"] + } + assert {recipe["benchmark"]["env"]["H3_RUNTIME"] for recipe in recipes} == { + "sglang", "vllm-omni" + } + + @pytest.mark.parametrize("edits", [ {"H3_SET": "missing"}, {"H3_SET": "../release-v1"}, diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 77a4b434b4..d35e5056f7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10118,3 +10118,13 @@ - "Run both H200 MiniMax-H3 engines at concurrency 1 on the same selected YAML workload: four existing prompts crossed with five cited seeds, 20 measured clips and one warmup." - "Replace the H200 SGLang four-clip c1/c2/c4 recipe with the selected two-GPU c1 workload, and add the two-GPU vLLM-Omni FL2VA recipe with the pinned v0.30.0 image digest and TP2/VAE tile settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4243 + +- config-keys: + - wan22-bf16-h200-sglang-video + - wan22-bf16-h200-vllm-omni-video + scenario-type: + - video-generation + description: + - "Add eight-GPU H200 SGLang and vLLM-Omni deployments for Wan2.2-T2V-A14B at concurrency 1 on the same selected YAML workload: 20 measured video-only clips and one warmup per engine." + - "Pin the model revision and vLLM-Omni v0.30.0 image digest; use SGLang TP1/Ulysses8 and vLLM-Omni HSDP/CFG2/USP4/VAE8 while preserving the shared generation and native power checks." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3733