Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion packages/user-intent-kit/src/host-telemetry.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0

/**
* Host vitals for the UIK agent record (CPU load, temperature, power).
* Host vitals for the UIK agent record (CPU load, temperature, power, GPU).
*
* Contract: every field is OPTIONAL and a value we cannot read is OMITTED,
* never zeroed. A zero renders in the dashboard as a real reading — an idle
Expand Down Expand Up @@ -373,6 +373,37 @@ function readLinuxTempC(sources) {
return { tempC: Math.round(hottest * 10) / 10, limitC };
}

/**
* The GPU's own vitals, from `nvidia-smi`, Linux only.
*
* Board thermal zones say nothing about how busy the GPU is: a GPU box at 95%
* and one at 5% can read the same zone temperature, so a card needs the GPU's
* own load to explain its heat (petrus, 2 Nov 2026). Each field is checked on
* its own, since nvidia-smi prints "[N/A]" for whatever a board does not
* expose (GB10 has no memory clock, for one). The first GPU is reported; every
* box in the fleet has one.
*
* @returns {{gpu_pct?: number, gpu_temp_c?: number, gpu_mhz?: number, gpu_w?: number}}
*/
function readNvidiaGpu(sources) {
const line = sources.run('/usr/bin/nvidia-smi', [
'--query-gpu=utilization.gpu,temperature.gpu,clocks.sm,power.draw',
'--format=csv,noheader,nounits',
]).split('\n')[0];
if (!line) return {};

const [pct, tempC, mhz, watts] = line.split(',').map((field) => {
const text = field.trim();
return /^-?\d+(\.\d+)?$/.test(text) ? Number(text) : NaN;
});
const out = {};
if (pct >= 0 && pct <= 100) out.gpu_pct = Math.round(pct);
if (tempC >= TEMP_MIN_C && tempC <= TEMP_MAX_C) out.gpu_temp_c = Math.round(tempC * 10) / 10;
if (mhz > 0) out.gpu_mhz = Math.round(mhz);
if (watts > 0) out.gpu_w = Math.round(watts * 10) / 10;
return out;
}

/**
* Collect what this host can actually report.
*
Expand Down Expand Up @@ -426,6 +457,12 @@ export function collectHostTelemetry({ machine, kind, model, sources = defaultSo
// No thermal zones exposed; publish without a temperature.
}

try {
if (sources.platform() === 'linux') Object.assign(host, readNvidiaGpu(sources));
} catch {
// No NVIDIA GPU, or no driver: publish without GPU fields.
}

try {
const hw = readHardware(sources);
if (hw) host.hw = hw;
Expand Down
39 changes: 39 additions & 0 deletions packages/user-intent-kit/test/host-telemetry.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -394,3 +394,42 @@ test('a nonsense critical point is dropped rather than published', () => {
assert.ok(!('temp_limit_c' in host), `published nonsense limit ${bad}`);
}
});

// --- the GPU's own vitals, from nvidia-smi ---

const NVSMI = '/usr/bin/nvidia-smi --query-gpu=utilization.gpu,temperature.gpu,clocks.sm,power.draw '
+ '--format=csv,noheader,nounits';

test('publishes GPU load, temperature, clock and power from nvidia-smi', () => {
const host = collectHostTelemetry({ sources: sources({ commands: { [NVSMI]: '95, 72, 2184, 27.64' } }) });
assert.equal(host.gpu_pct, 95);
assert.equal(host.gpu_temp_c, 72);
assert.equal(host.gpu_mhz, 2184);
assert.equal(host.gpu_w, 27.6);
});

test('a GPU field nvidia-smi cannot read is omitted, the rest still published', () => {
const host = collectHostTelemetry({ sources: sources({ commands: { [NVSMI]: '40, 61, 1500, [N/A]' } }) });
assert.equal(host.gpu_pct, 40);
assert.ok(!('gpu_w' in host), 'published an unreadable power draw');
});

test('an idle GPU publishes 0% load, a real reading', () => {
const host = collectHostTelemetry({ sources: sources({ commands: { [NVSMI]: '0, 45, 208, 4.1' } }) });
assert.equal(host.gpu_pct, 0);
});

test('a host without nvidia-smi publishes no GPU fields', () => {
const host = collectHostTelemetry({ sources: sources() });
for (const key of ['gpu_pct', 'gpu_temp_c', 'gpu_mhz', 'gpu_w']) assert.ok(!(key in host), `published ${key}`);
});

test('macOS never asks nvidia-smi', () => {
const host = collectHostTelemetry({ sources: sources({ platform: 'darwin', commands: { [NVSMI]: '95, 72, 2184, 27' } }) });
assert.ok(!('gpu_pct' in host));
});

test('implausible GPU readings are omitted', () => {
const host = collectHostTelemetry({ sources: sources({ commands: { [NVSMI]: '140, 999, -5, abc' } }) });
for (const key of ['gpu_pct', 'gpu_temp_c', 'gpu_mhz', 'gpu_w']) assert.ok(!(key in host), `published ${key}`);
});
Loading