[1/4] [Power] feat: validate single-node GPU energy metrics / 校验单节点 GPU 能耗指标

.github/workflows/benchmark-tmpl.yml

@@ -129,6 +129,11 @@ on:
         required: false
         type: string
         default: '3600'
+      require-power:
+        description: "Fail fixed-sequence result processing when GPU power is invalid"
+        required: false
+        type: boolean
+        default: false
       agentx-fast:
         description: "Use one warmup request per AgentX lane and a 20-minute profile"
         required: false
@@ -179,6 +184,7 @@ env:
   KV_P2P_TRANSFER: ${{ inputs.kv-p2p-transfer }}
   TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }}
   DURATION: ${{ inputs.duration }}
+  REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
   AIPERF_EXPERIMENTAL_FAST: ${{ inputs.agentx-fast && '1' || '0' }}
   EVAL_LIMIT: ${{ inputs.eval-limit }}
   SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }}
@@ -359,6 +365,18 @@ jobs:
           path: gpu_metrics.csv
           if-no-files-found: ignore
 
+      - name: Upload power audit bundle
+        if: ${{ always() && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: power_audit_${{ env.RESULT_FILENAME }}
+          path: |
+            ${{ env.RESULT_FILENAME }}.json
+            agg_${{ env.RESULT_FILENAME }}.json
+            gpu_metrics.csv
+            power_validation_${{ env.RESULT_FILENAME }}.json
+          if-no-files-found: ignore
+
       - name: Upload eval results (if any)
         if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/e2e-tests.yml

@@ -25,6 +25,11 @@ on:
                 required: false
                 type: string
                 default: ""
+            require-power:
+                description: "Fail single-node fixed-sequence jobs when GPU power is invalid"
+                required: false
+                type: boolean
+                default: false
             agentx-fast:
                 description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile."
                 required: false
@@ -95,6 +100,11 @@ on:
                 required: false
                 type: string
                 default: ""
+            require-power:
+                description: "Fail single-node fixed-sequence jobs when GPU power is invalid"
+                required: false
+                type: boolean
+                default: false
             agentx-fast:
                 description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile."
                 required: false
@@ -531,6 +541,7 @@ jobs:
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
             run-eval: false
+            require-power: ${{ inputs.require-power }}
             ref: ${{ inputs.ref }}
 
     test-sweep-evals:

.github/workflows/test-process-result.yml

@@ -3,15 +3,21 @@ name: Test Process Result
 on:
   pull_request:
     paths:
+      - '.github/workflows/benchmark-tmpl.yml'
+      - '.github/workflows/e2e-tests.yml'
+      - '.github/workflows/test-process-result.yml'
+      - 'benchmarks/benchmark_lib.sh'
+      - 'utils/aggregate_power.py'
+      - 'utils/bench_serving/benchmark_serving.py'
       - 'utils/process_result.py'
+      - 'utils/test_aggregate_power.py'
       - 'utils/test_process_result.py'
 
 permissions:
   contents: read
 
 jobs:
   test:
-    if: github.event.pull_request.draft != true
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -33,4 +39,4 @@ jobs:
       - name: Run pytest
         run: |
           cd utils
-          pytest test_process_result.py -v
+          python -m pytest test_aggregate_power.py test_process_result.py -v

benchmarks/benchmark_lib.sh

@@ -109,7 +109,9 @@ unset _benchmark_caller
 # --------------------------------
 
 GPU_MONITOR_PID=""
-GPU_METRICS_CSV="/workspace/gpu_metrics.csv"
+GPU_MONITOR_VENDOR=""
+GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}"
+NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory"
 export GPU_METRICS_CSV
 
 # Start background GPU monitoring that logs metrics every second to CSV.
@@ -131,18 +133,21 @@ start_gpu_monitor() {
     export GPU_METRICS_CSV
 
     if command -v nvidia-smi &>/dev/null; then
-        nvidia-smi --query-gpu=timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory \
+        GPU_MONITOR_VENDOR="nvidia"
+        nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
             --format=csv -l "$interval" > "$output" 2>/dev/null &
         GPU_MONITOR_PID=$!
         echo "[GPU Monitor] Started NVIDIA (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)"
     elif command -v amd-smi &>/dev/null; then
+        GPU_MONITOR_VENDOR="amd"
         # Use amd-smi native watch mode (-w) which includes timestamps automatically.
         # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated headers.
         amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \
             | awk '/^timestamp,/{if(!h){print;h=1};next} h{print}' > "$output" &
         GPU_MONITOR_PID=$!
         echo "[GPU Monitor] Started AMD (PID=$GPU_MONITOR_PID, interval=${interval}s, output=$output)"
     else
+        GPU_MONITOR_VENDOR=""
         echo "[GPU Monitor] No GPU monitoring tool found (nvidia-smi or amd-smi), skipping"
         return 0
     fi
@@ -153,6 +158,33 @@ stop_gpu_monitor() {
     if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then
         kill "$GPU_MONITOR_PID" 2>/dev/null
         wait "$GPU_MONITOR_PID" 2>/dev/null || true
+        # benchmark_end_time_unix is recorded shortly before the benchmark
+        # process exits. For the NVIDIA PR1 canary, append one post-exit sample
+        # so boundary interpolation is deterministic even when the run ends
+        # between 1 Hz monitor ticks. AMD one-shot CSV schemas vary by amd-smi
+        # version, so strict AMD lifecycle validation remains follow-up work.
+        case "$GPU_MONITOR_VENDOR" in
+            nvidia)
+                local append_final_nvidia_sample=true
+                local repaired_metrics="${GPU_METRICS_CSV}.repair.$$"
+                if [[ -s "$GPU_METRICS_CSV" ]] &&
+                    ! tail -c 1 "$GPU_METRICS_CSV" | grep -q '^$'; then
+                    if sed '$d' "$GPU_METRICS_CSV" > "$repaired_metrics" &&
+                        mv "$repaired_metrics" "$GPU_METRICS_CSV"; then
+                        echo "[GPU Monitor] Dropped truncated trailing sample"
+                    else
+                        rm -f "$repaired_metrics"
+                        append_final_nvidia_sample=false
+                        echo "[GPU Monitor] Warning: could not repair truncated trailing sample" >&2
+                    fi
+                fi
+                if [[ "$append_final_nvidia_sample" == true ]]; then
+                    nvidia-smi --query-gpu="$NVIDIA_GPU_MONITOR_QUERY" \
+                        --format=csv,noheader >> "$GPU_METRICS_CSV" 2>/dev/null ||
+                        echo "[GPU Monitor] Warning: final NVIDIA sample failed" >&2
+                fi
+                ;;
+        esac
         echo "[GPU Monitor] Stopped (PID=$GPU_MONITOR_PID)"
         if [[ -f "$GPU_METRICS_CSV" ]]; then
             local lines
@@ -161,6 +193,7 @@ stop_gpu_monitor() {
         fi
     fi
     GPU_MONITOR_PID=""
+    GPU_MONITOR_VENDOR=""
 }
 
 # Block until the GPUs have released a prior job's memory before starting a run.