diff --git a/docs/API.md b/docs/API.md index 83804a3f5..eb60701f3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,5 +1,16 @@ # Ocean Node Api +## Address casing + +Every EVM address you send (`consumerAddress`, `owner`, `address`, `decrypterAddress`, +`dataNftAddress`, `publisherAddress`, `consumerAddrs`, `additionalViewers`) is accepted in **any +casing** — checksummed (EIP-55), all-lowercase, all-uppercase — and canonicalized to its checksummed +form by the node before it is used as a lookup key or compared against an owner. A lowercased +address therefore matches the same jobs, services and buckets as the checksummed one. + +Signatures are unaffected: the node verifies the signed message against the casing you actually +signed, so clients that build the message from a lowercase address keep working. + --- ## State DDO @@ -1577,11 +1588,30 @@ returns job status Required at least one of the following parameters: -| name | type | required | description | -| --------------- | ------ | -------- | ------------------------------------ | -| consumerAddress | string | | consumer address to use as filter | -| jobId | string | | jobId address to use as filter | -| agreementId | string | | agreementId address to use as filter | +| name | type | required | description | +| --------------- | ------- | -------- | ------------------------------------------------------------------------------------------------------------ | +| consumerAddress | string | | consumer address to use as filter | +| jobId | string | | jobId address to use as filter | +| agreementId | string | | agreementId address to use as filter | +| includeMetrics | boolean | | override the runtime-metrics default (`true` = require them, `false` = never). See note below. | +| signature | string | | signature over `consumerAddress` + `nonce` + `command` (or an auth token) — authenticates the owner | +| nonce | string | | request nonce, paired with `signature` | + +**Runtime metrics are owner-only and returned BY DEFAULT.** If the request carries owner credentials +(`consumerAddress` plus `signature`/`nonce`, or an `Authorization` header token), each job owned by +(or shared with) that address comes back with a `runtimeMetrics` object — no flag needed. + +`includeMetrics` only overrides that default: + +| `includeMetrics` | behavior | +| --- | --- | +| omitted (default) | Metrics attached when owner credentials are present and valid. A request without credentials — the plain, unauthenticated status call — returns `200` with no metrics, exactly as before. Invalid credentials likewise just mean no metrics. | +| `true` | Metrics are **required**: missing `consumerAddress` answers `400`, failed authentication `401`. Use it when you want to know *why* metrics are absent instead of getting a silently trimmed response. | +| `false` | Metrics are never attached (and the node skips the auth round-trip). | + +Metrics never reach a non-owner, and are never part of the on-chain escrow claim proof. They are +best-effort and up to one sampling interval stale (see [compute.md](compute.md) and +`C2D_METRICS_INTERVAL_SECONDS` in [env.md](env.md)). #### Response @@ -1621,6 +1651,146 @@ Required at least one of the following parameters: ] ``` +When called with owner credentials, each owned job additionally carries a `runtimeMetrics` object +(see [The `runtimeMetrics` object](#the-runtimemetrics-object) below). + +--- + +### The `runtimeMetrics` object + +`runtimeMetrics` is an optional snapshot of live container stats, returned on +`COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS` to the **authenticated owner** of the job or service — +by default, without asking for it. It never reaches anyone else, and `includeMetrics=false` opts out. +Clients MUST treat every part as optional and render a field only when present. + +**Semantics clients should surface to users:** + +- **Best-effort & slightly stale.** Sampled on a fixed cadence (`C2D_METRICS_INTERVAL_SECONDS`, + default 10s), so values can be up to one interval old. `collectedAt` is the sample time — show it + (e.g. "as of 8s ago"). +- **May be missing entirely.** No snapshot yet (job just started), collection disabled on the node + (`C2D_METRICS_INTERVAL_SECONDS=0`), or a transient sampling failure ⇒ no `runtimeMetrics` field. + This is normal, not an error. +- **`null` vs absent for GPU numbers.** Inside a `gpu[]` entry, a `null` metric means "the backend + could not read it" — display as "n/a", never as `0`. +- **Bytes are raw bytes**; percentages are rounded to two decimals — memory/disk/GPU are `0–100`, + but CPU `usagePercent` can exceed `100` across multiple cores (see the CPU table); durations are + seconds; the final snapshot after a job/service ends carries the peak/exit values. + +#### Top-level fields + +| field | type | unit / notes | +| --------------- | ------- | ------------------------------------------------------------------------------------------------ | +| collectedAt | string | ISO-8601 timestamp of the sample | +| containerState | object | see below — status + structured exit info | +| cpu | object | see below | +| memory | object | see below | +| disk | object | see below | +| network | object? | `{ rxBytes, txBytes }`; **absent** when the container runs with no network (`NetworkMode: none`) | +| blockIO | object | `{ readBytes, writeBytes }` — cumulative disk I/O in bytes | +| pids | object | `{ current, limit }` — process/thread count vs the container PID limit (512) | +| gpu | array? | one entry per GPU the job/service holds; **absent** for CPU-only jobs or when GPU metrics are off | + +`containerState`: + +| field | type | notes | +| ------------ | -------- | ------------------------------------------------------------------------ | +| status | string | e.g. `running`, `exited` | +| startedAt | string? | ISO-8601 | +| finishedAt | string? | ISO-8601; present once the container has stopped | +| exitCode | number? | process exit code (present after exit) | +| oomKilled | boolean | `true` if the kernel OOM-killed the container | +| error | string? | Docker-reported error string, if any | +| restartCount | number | container restarts | +| health | string? | Docker HEALTHCHECK status when the image defines one (e.g. `healthy`) | + +`cpu`: + +| field | type | unit / notes | +| ---------------------- | ------ | ------------------------------------------------------------------------------- | +| usagePercent | number | % of one host CPU-second per wall-second (docker-stats formula), `0–N×100` | +| allocated | number | CPU cores requested by the job (`0` when unconstrained) | +| usagePercentOfAllocated| number | `usagePercent / allocated` — "how saturated is what you paid for" (`0` if alloc 0)| +| cumulativeSeconds | number | total CPU-seconds consumed since start (monotonic; billing-grade) | +| throttledPeriods | number | CFS quota throttling events — high ⇒ the CPU request is too small | +| throttledSeconds | number | total time throttled, seconds | + +`memory`: + +| field | type | unit / notes | +| -------------- | ------ | -------------------------------------------------------- | +| usageBytes | number | working-set bytes (`usage − inactive_file`, cgroup v2) | +| limitBytes | number | memory limit (= allocated RAM) in bytes | +| usagePercent | number | `usageBytes / limitBytes × 100` | +| peakUsageBytes | number | max `usageBytes` observed across samples | + +`disk`: + +| field | type | unit / notes | +| ------------ | ------- | ---------------------------------------------------------------------------------- | +| usedBytes | number | compute jobs: bytes written under `/` (excludes base image); services: writable layer | +| quotaBytes | number? | present only for jobs with a `disk` resource | +| usagePercent | number? | present only when `quotaBytes` is known | + +`gpu[]` entry: + +| field | type | unit / notes | +| ------------------ | --------------- | --------------------------------------------------------------------- | +| resourceId | string | the requested resource id (`gpu0`, `gpu1`, …) — maps the entry to a device | +| vendor | string | `nvidia` (only NVIDIA is emitted today; `amd`/`intel` reserved) | +| utilizationPercent | number \| null | GPU busy % (`null` = unreadable) | +| memoryUsedBytes | number \| null | VRAM used | +| memoryTotalBytes | number \| null | total VRAM | +| temperatureC | number? | °C, when available | +| powerWatts | number? | current draw, W, when available | +| shared | boolean? | `true` ⇒ device is shareable and the number may include other jobs | + +> Note: the node also keeps an internal delta accumulator on the stored snapshot; it is stripped +> from the response and clients will never see it. + +#### Example `runtimeMetrics` + +```json +{ + "collectedAt": "2026-07-29T12:00:10.000Z", + "containerState": { + "status": "running", + "startedAt": "2026-07-29T11:59:30.000Z", + "oomKilled": false, + "restartCount": 0 + }, + "cpu": { + "usagePercent": 182.4, + "allocated": 2, + "usagePercentOfAllocated": 91.2, + "cumulativeSeconds": 73.1, + "throttledPeriods": 12, + "throttledSeconds": 0.4 + }, + "memory": { + "usageBytes": 734003200, + "limitBytes": 1073741824, + "usagePercent": 68.36, + "peakUsageBytes": 812345678 + }, + "disk": { "usedBytes": 524288000, "quotaBytes": 10737418240, "usagePercent": 4.88 }, + "network": { "rxBytes": 10485760, "txBytes": 2097152 }, + "blockIO": { "readBytes": 41943040, "writeBytes": 8388608 }, + "pids": { "current": 24, "limit": 512 }, + "gpu": [ + { + "resourceId": "gpu0", + "vendor": "nvidia", + "utilizationPercent": 77, + "memoryUsedBytes": 1073741824, + "memoryTotalBytes": 3221225472, + "temperatureC": 55, + "powerWatts": 90 + } + ] +} +``` + ### `HTTP` GET /api/services/computeResult ### `P2P` command: getComputeResult @@ -2002,16 +2172,21 @@ by the authenticated `consumerAddress` are returned. #### Query Parameters -| name | type | required | description | -| --------------- | ------ | -------- | ----------- | -| consumerAddress | string | v | owner address | -| nonce | string | v | request nonce | -| signature | string | v | signed message (or use an `Authorization` auth-token header) | -| serviceId | string | | filter to a single service; omit to list all owned services | +| name | type | required | description | +| --------------- | ------- | -------- | ----------- | +| consumerAddress | string | v | owner address | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| serviceId | string | | filter to a single service; omit to list all owned services | +| includeMetrics | boolean | | runtime metrics (`runtimeMetrics`) are included by default; pass `false` to omit them | #### Response (200) -Array of `ServiceJob` (with `userData` stripped). +Array of `ServiceJob` (with `userData` stripped). Each entry also carries a sanitized +`runtimeMetrics` object — see [The `runtimeMetrics` object](#the-runtimemetrics-object) for its full +structure. Included by default here because this command is already authenticated and owner-scoped +(pass `includeMetrics=false` to omit); the node-wide `serviceList` never returns metrics. Metrics are +best-effort (see [compute.md](compute.md) and `C2D_METRICS_INTERVAL_SECONDS` in [env.md](env.md)). --- diff --git a/docs/Logs.md b/docs/Logs.md index 0b746d40f..faea49cdb 100644 --- a/docs/Logs.md +++ b/docs/Logs.md @@ -100,3 +100,60 @@ npm run logs ``` npm run logs http://localhost:8000 "2023-11-01T00:00:00Z" "2023-11-30T23:59:59Z" 50 "http" "info" ``` + +## Compute/Service runtime metrics in the logs + +While compute jobs and services run, the node samples live container stats (CPU, RAM, disk, +network, block I/O, PIDs, exit info, GPU) every `C2D_METRICS_INTERVAL_SECONDS` — see +[env.md](env.md). Every one of those lines is logged at **debug** level by the `CORE` module +with a `[metrics]` tag, so one filter shows the whole picture: + +```bash +# everything metrics-related (needs LOG_LEVEL=debug) +grep '\[metrics\]' logs/*.log + +# just the engine-wide roll-up: one line per sampling interval +grep '\[metrics\] summary' logs/*.log + +# just the workloads close to a limit (mem/disk/pids/cpu-throttling) +grep '\[metrics\] pressure' logs/*.log + +# one specific job or service, sample by sample +grep '\[metrics\] job 88ee41c8' logs/*.log +``` + +What each tag means: + +| line | when | what it tells you | +| --- | --- | --- | +| `[metrics] C2D Engine : sampling every Ns` / `collection DISABLED` | engine start | whether metrics are being collected at all — the first thing to check when a job shows no `runtimeMetrics` | +| `[metrics] summary engine : …` | once per interval | totals across every sampled job/service on that engine: cpu % of host, cores allocated, memory used vs allocated, disk, network, GPU count, how many containers are cpu-throttled, and the age of the oldest sample | +| `[metrics] pressure job\|service : …` | once per interval, only when relevant | that workload is ≥90% of its memory limit (OOM-kill risk), ≥90% of its disk quota (stop risk), ≥80% of its PID limit, or is being cpu-throttled (undersized `cpu` request) | +| `[metrics] job\|service : cpu … mem … disk … pids … net … blkio … state …` | per sample | the full `docker stats` view of that container, plus throttling, peak memory, disk vs quota, exit info and GPU. `[final]` marks the last snapshot taken before teardown | +| `[metrics] job : first snapshot …` | first sample of a job | reminder that `cpu` reads 0% until the second sample (deltas need two samples) | +| `[metrics] … dropping sample …` / `collection failed …` | on failure | why a snapshot is missing: the container vanished, or a lifecycle operation was in flight | +| `[metrics] gpu: …` and `GPU metrics (nvidia): …` | on failure | no GPU numbers, and why. Each message is explained — with its fix — in [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) | + +These are pure diagnostics — collection is best-effort and never affects job or service +execution. The same numbers are available over the API to the owner of a job/service +(`runtimeMetrics` on `COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS`, see [API.md](API.md)). + +### When a status response has no `runtimeMetrics` + +Snapshots are persisted on the job record, so you can check the stored state directly instead +of guessing whether they were never written or dropped on the way out: + +```bash +npm run job-metrics # the 5 most recent jobs +npm run job-metrics -- 018c0121 # one job (full id or any trailing part) +``` + +For each job it prints whether the row holds a snapshot, its values, and whether the delta +accumulator that CPU % needs is there. Read-only; run it from the node's working directory. + +- **`runtimeMetrics: ABSENT`** — nothing was ever stored for that job, so the API cannot return + it and `cpu usagePercent` stays 0 (each sample would be a "first sample"). The node also warns + about this itself: `[metrics] job : no previous snapshot after Ns of runtime`. +- **present, but missing from the response** — the caller was not recognised as the owner. + Metrics only go to the job's owner (or an `additionalViewers` address), proven by + `consumerAddress` + signature/nonce or an `Authorization` token issued to that same address. diff --git a/docs/Ocean Node.postman_collection.json b/docs/Ocean Node.postman_collection.json index c2a91cf70..df42f4418 100644 --- a/docs/Ocean Node.postman_collection.json +++ b/docs/Ocean Node.postman_collection.json @@ -653,7 +653,7 @@ "method": "GET", "header": [], "url": { - "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&jobId={{jobId}}&agreementId=", + "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&jobId={{jobId}}&agreementId=&signature={{signature}}&nonce={{nonce}}", "host": [ "{{baseUrl}}" ], @@ -675,6 +675,24 @@ "key": "agreementId", "value": "" }, + { + "key": "includeMetrics", + "value": "false", + "description": "Runtime metrics (runtimeMetrics) come back BY DEFAULT on jobs you own — provide consumerAddress + signature/nonce (or an Authorization token). Set 'true' to require them (400/401 when unauthenticated instead of no metrics), 'false' to omit them.", + "disabled": true + }, + { + "key": "signature", + "value": "{{signature}}", + "description": "Authenticates the owner: signature over consumerAddress + nonce + command. Supply it (with nonce) to receive runtimeMetrics; required when includeMetrics=true.", + "disabled": true + }, + { + "key": "nonce", + "value": "{{nonce}}", + "description": "Request nonce, paired with signature. Required when includeMetrics=true.", + "disabled": true + }, { "key": "node", "value": "", @@ -682,7 +700,7 @@ } ] }, - "description": "Get the status of a compute job." + "description": "Get the status of a compute job.\n\n`runtimeMetrics` (Docker/NVML stats) is owner-only and returned BY DEFAULT: authenticate as the owner (signature+nonce, or an `Authorization` token header, together with `consumerAddress`) and every job you own comes back with a sanitized `runtimeMetrics` object. An unauthenticated status call returns no metrics. Use `includeMetrics=true` to make them mandatory (400/401 instead of a silently trimmed response), or `includeMetrics=false` to opt out. See the API docs for the field structure." } }, { @@ -943,6 +961,12 @@ "key": "serviceId", "value": "{{serviceId}}" }, + { + "key": "includeMetrics", + "value": "false", + "description": "A sanitized runtimeMetrics object (Docker/NVML stats) is included on each returned service BY DEFAULT — safe because this call is already authenticated + owner-scoped. Set 'false' to omit it.", + "disabled": true + }, { "key": "node", "value": "", @@ -950,7 +974,7 @@ } ] }, - "description": "Read service status/endpoints. Authenticated and owner-scoped. Omit serviceId to list all owned services." + "description": "Read service status/endpoints. Authenticated and owner-scoped. Omit serviceId to list all owned services.\n\nA sanitized `runtimeMetrics` object per service is included BY DEFAULT (this command is already authenticated and owner-scoped); pass `includeMetrics=false` to omit it. See the API docs for the field structure." } }, { @@ -2423,4 +2447,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/docs/compute.md b/docs/compute.md index 41616f52c..742c4ca80 100644 --- a/docs/compute.md +++ b/docs/compute.md @@ -215,6 +215,19 @@ The environment references it by `id`. ] ``` +> **GPU runtime metrics (NVIDIA).** When `GPU_METRICS` is `auto` (the default), the node +> records per-GPU utilization and memory for running jobs/services alongside their container +> metrics, sampled every `C2D_METRICS_INTERVAL_SECONDS`. This uses NVML via the optional +> `koffi` dependency and requires `libnvidia-ml.so.1` to be reachable by the node process (the +> NVIDIA driver provides it; in a containerized node, mount it or use the NVIDIA container +> toolkit). A job holding several GPUs gets one metrics entry per device, keyed by the +> resource id it requested. If NVML is unavailable, GPU metrics are skipped (one warning, then +> silence) while container metrics continue — see +> [Troubleshooting GPU metrics](#troubleshooting-gpu-metrics) for what each warning means and +> how to fix it. AMD and Intel GPU metrics are not yet collected. GPU metrics are returned only +> to the owner of the job/service, alongside its container metrics. Set `GPU_METRICS=off` to +> disable. + ### AMD Radeon (ROCm) Install [ROCm](https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/wsl/install-radeon.html), @@ -309,6 +322,76 @@ simultaneously. The engine tracks `inUse` for visibility but never blocks alloca > `shareable: true` is **not** allowed on `type: "gpu"` or `type: "fpga"` — the node refuses > to start. GPUs and FPGAs require exclusive per-job access. +### Troubleshooting GPU metrics + +GPU metrics are strictly best-effort. When they cannot be collected, jobs and services are +never affected and container-level metrics (cpu, memory, disk, network, block I/O, PIDs) keep +being collected — the snapshot just has no `gpu[]` entry. + +All messages come from the `CORE` module. The NVML load/init failures below are logged at +**warn**, so they are visible at the default log level, and only once per process (the probe +result is cached). The `[metrics] gpu:` lines are **debug**, so they need `LOG_LEVEL=debug`; +their frequency is noted per message. + +#### `could not bind libnvidia-ml.so.1 — NVIDIA GPU metrics disabled (Failed to load shared library: cannot open shared object file: No such file or directory)` + +The node found the `koffi` FFI but the **dynamic loader could not find the NVIDIA driver's NVML +library** anywhere in its search path. The message text after the dash comes straight from +`dlopen`. Causes, most common first: + +| # | Cause | Fix | +|---|---|---| +| 1 | **The node itself runs in a container without the driver injected.** The node does not need GPU access to *run* GPU jobs — it drives the Docker socket, and the NVIDIA Container Toolkit injects driver libraries into the **job** containers it starts, not into the node's own container. So `nvidia-smi` works on the host, GPU jobs run fine, and NVML is still missing inside the node container. | Give the node container the driver's `utility` capability — see the snippet below | +| 2 | **Host install the loader does not know about.** Driver present but in a non-standard prefix, so `ldconfig` never cached it. | Add an `/etc/ld.so.conf.d/*.conf` entry and run `ldconfig`, or set `LD_LIBRARY_PATH` in the node's process environment (e.g. the pm2 / systemd unit) | +| 3 | **WSL2.** The driver libraries live in `/usr/lib/wsl/lib`, which is not always on the loader path of the node's process. | `LD_LIBRARY_PATH=/usr/lib/wsl/lib`. Note that on WSL some NVML queries return NOT_SUPPORTED — utilization and memory usually work, power and temperature often do not, and are reported as `null` | +| 4 | **Missing soname symlink.** Only `libnvidia-ml.so.` exists, without the `libnvidia-ml.so.1` link — typical after a manual driver copy. | Run `ldconfig` to recreate it | +| 5 | **The host has no NVIDIA driver at all** — e.g. an AMD/Intel GPU whose resource is declared `"platform": "nvidia"`, or a leftover `gpu` resource on a CPU-only machine. | Fix the resource's `platform`, or set `GPU_METRICS=off` to stop probing | + +Giving a containerized node access to NVML (cause 1) — `utility` is the capability that injects +NVML and `nvidia-smi`; it does **not** reserve the GPU for the node or take it away from jobs: + +```yaml +# docker compose +services: + ocean-node: + runtime: nvidia # or deploy.resources.reservations.devices with capabilities: [gpu] + environment: + NVIDIA_VISIBLE_DEVICES: all + NVIDIA_DRIVER_CAPABILITIES: utility +``` + +```bash +# docker run +docker run --gpus all -e NVIDIA_DRIVER_CAPABILITIES=utility ... +``` + +Triage — if the host commands succeed and the in-container ones come back empty, it is cause 1: + +```bash +# on the host +nvidia-smi +ldconfig -p | grep libnvidia-ml +ls -l /usr/lib/x86_64-linux-gnu/libnvidia-ml.so* /usr/lib/wsl/lib/libnvidia-ml.so* 2>/dev/null + +# inside the node's container +docker exec sh -lc 'ldconfig -p | grep nvidia-ml; nvidia-smi || echo "no nvidia-smi here"' +``` + +#### Other GPU metrics messages + +| Message | Meaning | +|---|---| +| `koffi FFI not available — NVIDIA GPU metrics disabled` | The optional `koffi` dependency is not installed. `npm install` in the node directory installs it; a `--omit=optional` install skips it | +| `nvmlInit failed (code N)` / `nvmlInit threw` | The library loaded but NVML would not initialize — usually a driver/library version mismatch (reinstall the driver so userspace matches the kernel module), or missing permission on `/dev/nvidia*` | +| `[metrics] gpu: cannot determine the vendor of resource ""` | That GPU resource has neither `"platform"` (`nvidia` \| `amd` \| `intel`) nor `init.deviceRequests.Driver`, so no backend could be chosen. Set `platform` on the resource. Logged once per resource id | +| `[metrics] gpu: backend not implemented yet` | AMD and Intel GPU metrics are not collected yet. The GPUs still work for compute; only their metrics are missing. Logged once per vendor | +| `[metrics] gpu: N GPU resource(s) held but no device metrics resolved` | The workload holds GPUs but nothing came back — the follow-on symptom of any of the above. Logged per sample, so it is the line to grep when GPU numbers are missing | +| `[metrics] gpu: collection failed: …` | An unexpected error during a sweep; logged per sweep. The previous sample is kept | + +Set `GPU_METRICS=off` to disable GPU collection entirely (see [env.md](env.md)); container-level +metrics are unaffected. For where these lines appear and how to filter them, see +[Logs.md](Logs.md#computeservice-runtime-metrics-in-the-logs). + ### Migration from the old format The old format placed hardware resources (`init`, `driverVersion`, etc.) inside diff --git a/docs/database.md b/docs/database.md index edf2ae830..04e04714c 100644 --- a/docs/database.md +++ b/docs/database.md @@ -41,3 +41,22 @@ To run Ocean Node with the appropriate database, you need to start Barge with sp ``` By specifying these flags, you can configure Ocean Node to work with either Typesense or Elasticsearch databases, depending on your requirements. + +## Runtime metrics on C2D job records + +While a compute job or Service-on-Demand container runs, the node periodically samples live +Docker (and, on NVIDIA hosts, NVML) metrics — CPU, RAM, disk usage vs quota, network, block +I/O, PID count, CPU throttling, memory peak, exit/OOM info, and per-GPU utilization/memory — +and stores the latest snapshot on the job record (inside the existing JSON `body` blob of the +SQLite `c2djobs` / `service_jobs` tables). A **final** snapshot is written at termination +(publishing results, quota kill, service stop/restart, or unexpected container death) so +peak/exit metrics remain queryable after the container is gone. No schema migration is needed; +pre-upgrade records simply lack the field. + +These snapshots are **owner-only**: they are stripped from the escrow claim proof and from every +response except the authenticated owner's own `COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS` (where +they are included by default — see [API.md](API.md)). Sampling cadence is controlled by +`C2D_METRICS_INTERVAL_SECONDS` (`0` disables it) and GPU collection by `GPU_METRICS` — see +[env.md](env.md). When metrics are missing, [Logs.md](Logs.md#when-a-status-response-has-no-runtimemetrics) +covers how to inspect what is actually stored, and [compute.md](compute.md#troubleshooting-gpu-metrics) +covers the GPU-specific warnings. diff --git a/docs/env.md b/docs/env.md index ab1995d92..92c572dee 100644 --- a/docs/env.md +++ b/docs/env.md @@ -129,6 +129,10 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `C2D_DOWNLOAD_TIMEOUT`: Timeout (in seconds) for pulling the algorithm docker image during a C2D job. If the pull exceeds this timeout, the job fails with `PullImageFailed` instead of getting stuck. Defaults to `900` (15 minutes). Example: `900` +- `C2D_METRICS_INTERVAL_SECONDS`: How often (in seconds) the node samples live Docker runtime metrics (CPU, RAM, disk, network, block I/O, PIDs, exit info — plus NVIDIA GPU utilization/memory) for running compute jobs and services, persisting a snapshot onto the job record in the C2D database. These metrics are **owner-only**: they are never included in the escrow claim proof and never returned to anyone but the authenticated owner of the job/service. To that owner they come back **by default** on `COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS` (no flag needed — see [API.md](API.md) for the `includeMetrics` override); an unauthenticated status call and the node-wide `serviceList` never return them. Set to `0` to disable collection entirely. Metrics are best-effort (up to one interval of staleness). Defaults to `10`. Example: `10` + +- `GPU_METRICS`: Controls the GPU metrics collector. `auto` (default) detects and enables the NVIDIA (NVML) backend when a GPU host is available; `off` disables GPU collection. Requires the optional `koffi` dependency and `libnvidia-ml.so.1` reachable **by the node process** — note that a containerized node does not get the NVIDIA driver libraries just because the host has them, so this is the usual reason GPU metrics are missing (`could not bind libnvidia-ml.so.1`); [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) lists every warning and its fix. If either is missing, GPU metrics are skipped (no `gpu` field) while container-level metrics continue. AMD and Intel backends are not yet implemented. Cadence reuses `C2D_METRICS_INTERVAL_SECONDS`. Defaults to `auto`. Example: `auto` + - `SERVICE_TEMPLATES_PATH`: Path to a folder of operator-published Service-on-Demand template files (`*.json`, validated against the template schema). The folder is re-read on every `serviceTemplates` request, so templates can be added, edited, or removed without restarting the node. Maps to the `serviceTemplatesPath` config field. Defaults to `databases/serviceTemplates/`. See the [Services guide](services.md). Example: `docs/serviceTemplates/` The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable is used to configure Docker-based compute environments in Ocean Node. For the full guide — resources, GPU setup, constraints and pricing — see [Compute Configuration](compute.md). diff --git a/package-lock.json b/package-lock.json index 0066e6c4d..907bf65e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,6 +100,9 @@ }, "engines": { "node": ">=22.13.0" + }, + "optionalDependencies": { + "koffi": "^2.9.0" } }, "node_modules/@achingbrain/http-parser-js": { @@ -13574,6 +13577,17 @@ "json-buffer": "3.0.1" } }, + "node_modules/koffi": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-2.16.3.tgz", + "integrity": "sha512-E9y1AsgYGlaxMhcZzHr8y96QF2U5XzA12GGVAfbWqIubTwPNMXQarfBzePNXHe0xtIEtNd6ifAv3GAKYGUeBAQ==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, "node_modules/kuler": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", diff --git a/package.json b/package.json index 0d268e2ca..51b1229bf 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,9 @@ "winston-transport": "^4.6.0", "zod": "^3.25.76" }, + "optionalDependencies": { + "koffi": "^2.9.0" + }, "devDependencies": { "@types/chai": "^4.3.10", "@types/cors": "^2.8.17", diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 74bd3e267..d1d531f05 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -238,6 +238,73 @@ export type DBComputeJobMetadata = { [key: string]: string | number | boolean } +// Per-GPU runtime metrics captured alongside the container snapshot. ONE entry per GPU +// resource the job/service holds — a single job may hold several GPUs simultaneously, and +// each is sampled independently. Only NVIDIA is emitted today (NVML backend); AMD/Intel +// land once their backends exist. `resourceId` maps the entry back to the requested +// resource id ('gpu0' / 'gpu1' …) so consumers can attribute each device. +export interface GpuMetricsSnapshot { + resourceId: string + vendor: 'nvidia' | 'amd' | 'intel' + utilizationPercent: number | null + memoryUsedBytes: number | null + memoryTotalBytes: number | null + temperatureC?: number + powerWatts?: number + shared?: boolean // true → device-level number, may include other jobs' load (shareable GPUs) +} + +// A best-effort, node-internal snapshot of everything the Docker engine (and, for NVIDIA, +// NVML) can cheaply tell us about a running container. Persisted onto the C2D job record +// (JSON body blob) and STRIPPED from every public response + the escrow claim proof — see +// omitDBComputeFieldsFromComputeJob / toPublicServiceJob. Covers the full `docker stats` +// column set plus what the CLI does not show (throttling, peaks, disk-vs-quota, exit info, +// GPU). Every field defaults defensively (cgroup v1/v2 drift, missing network, daemon +// hiccups) rather than throwing into the state-machine loop. +export interface ContainerMetricsSnapshot { + collectedAt: string // ISO timestamp of the sample + containerState: { + status: string // 'running' | 'exited' | ... + startedAt?: string + finishedAt?: string + exitCode?: number + oomKilled: boolean + error?: string + restartCount: number + health?: string // Docker HEALTHCHECK status, when the image defines one (services) + } + cpu: { + usagePercent: number // % of host CPU (docker CLI formula) + allocated: number // cores requested (job.resources 'cpu'); 0 when unconstrained + usagePercentOfAllocated: number // usagePercent / allocated (0 when allocated is 0) + cumulativeSeconds: number // total_usage ns → s (monotonic; billing-grade) + throttledPeriods: number // quota throttling — signals an undersized cpu request + throttledSeconds: number + } + memory: { + usageBytes: number // usage − inactive_file (cgroup v2 convention) + limitBytes: number // container limit (= allocated RAM) + usagePercent: number + peakUsageBytes: number // max across samples (we track it; cgroup v2 has no max_usage) + } + disk: { + usedBytes: number // jobs: du(/) − base image; services: SizeRw + quotaBytes?: number // jobs with a 'disk' resource + usagePercent?: number + } + network?: { rxBytes: number; txBytes: number } // absent when NetworkMode 'none' + blockIO: { readBytes: number; writeBytes: number } + pids: { current: number; limit: number } // vs PidsLimit 512 — fork-bomb signal + gpu?: GpuMetricsSnapshot[] + // Internal accumulator used to compute one-shot CPU deltas across samples. DB-only — + // stripped from every public response together with the snapshot itself. + prev?: { + cpuTotal: number // cpu_stats.cpu_usage.total_usage (ns) at the previous sample + systemCpu: number // cpu_stats.system_cpu_usage (ns) at the previous sample + sampledAt: string + } +} + export interface ComputeJobTerminationDetails { OOMKilled: boolean exitCode: number @@ -261,6 +328,14 @@ export interface ComputeJob { queueMaxWaitTime: number // max time in seconds a job can wait in the queue before being started } +// Public compute-job response shape: the base ComputeJob plus the OPTIONAL, sanitized runtime +// metrics that the owner status path may attach (never present by default, and never in the +// escrow proof). Keeps ComputeJob itself free of the field while giving the status path a typed +// return instead of an `any` cast. +export interface PublicComputeJob extends ComputeJob { + runtimeMetrics?: ContainerMetricsSnapshot +} + export interface ComputeOutputEncryption { encryptMethod: EncryptMethod.AES // in future we will support more ciphers key: string // AES symetric key @@ -343,6 +418,11 @@ export interface DBComputeJob extends ComputeJob { jobIdHash: string buildStartTimestamp?: string buildStopTimestamp?: string + // Best-effort Docker/NVML runtime metrics sampled while the container runs. DB-only: + // it lives on DBComputeJob (never on the public ComputeJob) and is additionally stripped + // at runtime by omitDBComputeFieldsFromComputeJob so it can never leak into a status + // response or the escrow claim proof. + runtimeMetrics?: ContainerMetricsSnapshot } // make sure we keep them both in sync diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index f859530b5..b84cd8b65 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -1,4 +1,8 @@ -import type { DBComputeJobPayment, ComputeResourceRequestWithPrice } from './C2D.js' +import type { + DBComputeJobPayment, + ComputeResourceRequestWithPrice, + ContainerMetricsSnapshot +} from './C2D.js' // ── Resource requirements ───────────────────────────────────────────── @@ -144,4 +148,7 @@ export interface ServiceJob { resources: ComputeResourceRequestWithPrice[] payment: DBComputeJobPayment // initial start payment extendPayments?: DBComputeJobPayment[] // one entry per successful SERVICE_EXTEND + // Best-effort Docker/NVML runtime metrics sampled while the service container runs. + // DB-only: stripped from every public response by toPublicServiceJob / toListedServiceJob. + runtimeMetrics?: ContainerMetricsSnapshot } diff --git a/src/@types/commands.ts b/src/@types/commands.ts index 4bc087c39..f9f5de77e 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -300,6 +300,15 @@ export interface ComputeGetStatusCommand extends Command { consumerAddress?: string jobId?: string agreementId?: string + // Runtime metrics (runtimeMetrics) on the returned jobs, honored ONLY for the authenticated + // owner (consumerAddress + signature/nonce, or an authorization token). + // undefined (default) ⇒ attached when the caller carries owner credentials, silently omitted + // otherwise (an unauthenticated status call behaves exactly as it always did); + // true ⇒ explicitly required: missing/invalid credentials answer 400/401 instead; + // false ⇒ never attached. + includeMetrics?: boolean + nonce?: string + signature?: string } export interface ValidateChainId { @@ -448,6 +457,9 @@ export interface ServiceGetStatusCommand extends Command { nonce: string signature: string serviceId?: string + // Runtime metrics (runtimeMetrics) on the returned services. Included BY DEFAULT — safe + // because this command is already authenticated + owner-scoped. Pass false to opt out. + includeMetrics?: boolean } // Node-wide service listing (SERVICE_LIST), shaped like GetJobsCommand. Authenticated diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index d6cf534f4..d8d6f7798 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -197,7 +197,8 @@ export abstract class C2DEngine { public abstract getComputeJobStatus( consumerAddress?: string, agreementId?: string, - jobId?: string + jobId?: string, + includeMetrics?: boolean ): Promise public abstract getComputeJobResult( diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 49e7d4a2e..e7b28fd73 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -24,7 +24,8 @@ import type { ComputeResourceKind, C2DEnvironmentConfig, ComputeResourcesPricingInfo, - EnvironmentResourceRef + EnvironmentResourceRef, + ContainerMetricsSnapshot } from '../../@types/C2D/C2D.js' import { BASE_CHAIN_ID, USDC_TOKEN_ADDRESS_BASE } from '../../utils/config.js' import { C2DEngine } from './compute_engine_base.js' @@ -59,6 +60,7 @@ import { decryptFilesObject, omitDBComputeFieldsFromComputeJob } from './index.j import { ValidateParams } from '../httpRoutes/validateCommands.js' import { Service } from '@oceanprotocol/ddo-js' import { getOceanTokenAddressForChain } from '../../utils/address.js' +import { includesAddress, sameAddress } from '../../utils/evmAddress.js' import { dockerRegistryAuth, OceanNodeConfig } from '../../@types/OceanNode.js' import { BaseFileObject, @@ -73,6 +75,16 @@ import { } from '../../@types/C2D/ServiceOnDemand.js' import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' import { resolveServiceImage } from './serviceResourceMatching.js' +import { + buildSnapshot, + describeSnapshot, + formatBytes, + getMetricsIntervalSeconds, + isMetricsCollectionEnabled, + isSnapshotStale, + sampleContainerMetrics +} from './containerMetrics.js' +import { GpuMetricsService } from './gpu/index.js' import { allocateHostPort, releaseHostPort, @@ -111,6 +123,12 @@ export class C2DEngineDocker extends C2DEngine { private cronTimer: any private cronTime: number = 2000 private jobImageSizes: Map = new Map() + // Best-effort GPU metrics collector (NVIDIA/NVML today). Lazily initializes its vendor + // backends on first use by a GPU job; a pure-CPU node never loads any GPU code. + private gpuMetrics: GpuMetricsService = new GpuMetricsService() + // Last time the engine-wide metrics roll-up was logged. The loop ticks every 2s but snapshots + // only refresh once per C2D_METRICS_INTERVAL_SECONDS, so the summary is throttled to match. + private lastMetricsSummaryAt: number = 0 private isInternalLoopRunning: boolean = false // Set true by stop() so a stopped engine cannot reschedule or run another InternalLoop pass. // Without this, an in-flight loop's finally → setNewTimer() resurrects the timer on a stopped @@ -460,6 +478,16 @@ export class C2DEngineDocker extends C2DEngine { } const consumerAddress = this.getKeyManager().getEthAddress() + // Say once, per engine, whether runtime-metrics sampling is on and how often: "no stats on + // my jobs" is otherwise indistinguishable from C2D_METRICS_INTERVAL_SECONDS=0. + CORE_LOGGER.debug( + isMetricsCollectionEnabled() + ? `[metrics] C2D Engine ${this.getC2DConfig().hash}: sampling every ` + + `${getMetricsIntervalSeconds()}s (C2D_METRICS_INTERVAL_SECONDS)` + : `[metrics] C2D Engine ${this.getC2DConfig().hash}: collection DISABLED ` + + `(C2D_METRICS_INTERVAL_SECONDS=0) — jobs and services will carry no runtimeMetrics` + ) + if (config.enableBenchmark) { if (supportedChains.includes(parseInt(BASE_CHAIN_ID))) { this.createBenchmarkEnvironment(sysinfo, envConfig) @@ -700,6 +728,9 @@ export class C2DEngineDocker extends C2DEngine { this.serviceOpPromises.clear() } this.isInternalLoopRunning = false + // Release any GPU metrics backends (e.g. nvmlShutdown). Best-effort, never throws; the + // optional chain also covers engines built without the constructor (e.g. test doubles). + this.gpuMetrics?.dispose() // Stop image cleanup timer if (this.imageCleanupTimer) { clearInterval(this.imageCleanupTimer) @@ -850,6 +881,9 @@ export class C2DEngineDocker extends C2DEngine { } const cost = this.getTotalCostOfJob(job.resources, minDuration, fee) + // INVARIANT: the claim proof MUST use the default (metrics-free) omit shape so it + // stays deterministic. Never pass { includeMetrics: true } here — runtimeMetrics is + // volatile and would make the signed proof unstable. const proof = JSON.stringify(omitDBComputeFieldsFromComputeJob(job)) jobsToClaim.push({ job, cost, proof }) } else { @@ -1637,7 +1671,8 @@ export class C2DEngineDocker extends C2DEngine { public override async getComputeJobStatus( consumerAddress?: string, agreementId?: string, - jobId?: string + jobId?: string, + includeMetrics: boolean = false ): Promise { const jobs = await this.db.getJob(jobId, agreementId, consumerAddress) if (jobs.length === 0) { @@ -1645,7 +1680,15 @@ export class C2DEngineDocker extends C2DEngine { } const statusResults = [] for (const job of jobs) { - const res: ComputeJob = omitDBComputeFieldsFromComputeJob(job) + // Runtime metrics are attached ONLY when the caller opted in AND is the verified owner + // (or an additional viewer) — the same ownership rule getComputeJobResult enforces. The + // handler verifies the signature for consumerAddress before setting includeMetrics. + const isOwner = + sameAddress(job.owner, consumerAddress) || + includesAddress(job.additionalViewers, consumerAddress) + const res: ComputeJob = omitDBComputeFieldsFromComputeJob(job, { + includeMetrics: includeMetrics && isOwner + }) // add results for algoLogs res.results = await this.getResults(job.jobId) statusResults.push(res) @@ -1666,8 +1709,8 @@ export class C2DEngineDocker extends C2DEngine { throw new Error(`Cannot find job with id ${jobId}`) } if ( - jobs[0].owner !== consumerAddress && - (!jobs[0].additionalViewers || !jobs[0].additionalViewers.includes(consumerAddress)) + !sameAddress(jobs[0].owner, consumerAddress) && + !includesAddress(jobs[0].additionalViewers, consumerAddress) ) { // consumerAddress is not the owner and not in additionalViewers throw new Error( @@ -1817,7 +1860,11 @@ export class C2DEngineDocker extends C2DEngine { // Service-on-Demand health check: catch Running services whose container died on its // own (crash, OOM, or Docker daemon down) instead of only noticing at expiresAt. - await this.checkRunningServices() + const runningServices = await this.checkRunningServices() + + // Roll this tick's snapshots into one engine-wide line (+ pressure lines) so an admin + // sees the whole live picture without correlating per-container samples by hand. + this.logMetricsSummary(jobs, runningServices) // Service-on-Demand starts: advance pending service jobs through the start pipeline. // Fire-and-forget (NOT awaited): an image pull can take minutes and must not block the @@ -2397,12 +2444,17 @@ export class C2DEngineDocker extends C2DEngine { } } } else { - const canContinue = await this.monitorDiskUsage(job) + const { canContinue, usedBytes } = await this.monitorDiskUsage(job) if (!canContinue) { // Job was terminated due to disk quota exceeded return } + // Sample live runtime metrics (throttled to C2D_METRICS_INTERVAL_SECONDS). Reuses + // the du(/) bytes just measured; transition paths below persist it via their own + // db.updateJob(), and the still-running path persists only when a new sample was taken. + const metricsSampled = await this.collectJobMetrics(job, usedBytes) + const timeNow = Date.now() / 1000 let expiry @@ -2448,6 +2500,9 @@ export class C2DEngineDocker extends C2DEngine { await this.db.updateJob(job) return } + // Container still running and not transitioning this tick: persist the fresh + // metrics snapshot if one was just taken (piggybacks on no other write here). + if (metricsSampled) await this.db.updateJob(job) } } } @@ -2481,6 +2536,11 @@ export class C2DEngineDocker extends C2DEngine { job.terminationDetails.OOMKilled = null job.terminationDetails.exitCode = null } + // Final runtime-metrics snapshot before the container is removed by cleanup: captures + // last CPU seconds, peak memory, final disk usage and exit/OOM info for postmortems. + // Reuse the last du(/)-measured disk usage (jobs don't inspect size:true, so buildSnapshot + // would otherwise fall back to an unset SizeRw and report 0). + await this.collectJobMetrics(job, job.runtimeMetrics?.disk?.usedBytes, true) const outputsArchivePath = this.getStoragePath() + '/' + job.jobId + '/data/outputs/outputs.tar' @@ -2811,7 +2871,8 @@ export class C2DEngineDocker extends C2DEngine { // Wait for container filesystem to stabilize await new Promise((resolve) => setTimeout(resolve, 3000)) - const actualBaseSize = await this.getContainerDiskUsage(container.id, '/') + // null (unmeasurable) → cache 0 so the quota check counts the whole container, as before. + const actualBaseSize = (await this.getContainerDiskUsage(container.id, '/')) ?? 0 this.jobImageSizes.set(job.jobId, actualBaseSize) CORE_LOGGER.info( @@ -2828,10 +2889,13 @@ export class C2DEngineDocker extends C2DEngine { } } + // Returns the measured bytes, or null when the container cannot be measured (not running, + // gone, unreadable `du` output). null means "unknown" — distinct from a real 0 — so callers + // never report an unmeasurable container as "0 bytes used". private async getContainerDiskUsage( containerName: string, path: string = '/data' - ): Promise { + ): Promise { try { const container = this.docker.getContainer(containerName) const containerInfo = await container.inspect() @@ -2839,7 +2903,7 @@ export class C2DEngineDocker extends C2DEngine { CORE_LOGGER.debug( `Container ${containerName} is not running, cannot check disk usage` ) - return 0 + return null } const exec = await container.exec({ @@ -2858,21 +2922,35 @@ export class C2DEngineDocker extends C2DEngine { const output = Buffer.concat(chunks).toString() const match = output.match(/(\d+)\s/) - return match ? parseInt(match[1], 10) : 0 + return match ? parseInt(match[1], 10) : null } catch (error) { CORE_LOGGER.error( `Failed to get container disk usage for ${containerName}: ${error.message}` ) - return 0 + return null } } - private async monitorDiskUsage(job: DBComputeJob): Promise { + // Returns whether the job may keep running, plus the measured algorithm disk usage in + // bytes (undefined when there is no disk quota to measure against). The byte figure is + // reused by the metrics collector so it does not run a second `du`. + private async monitorDiskUsage( + job: DBComputeJob + ): Promise<{ canContinue: boolean; usedBytes?: number }> { const diskQuota = this.getDiskQuota(job) - if (diskQuota <= 0) return true + if (diskQuota <= 0) return { canContinue: true } const containerName = job.jobId + '-algoritm' const totalUsage = await this.getContainerDiskUsage(containerName, '/') + if (totalUsage === null) { + // Unmeasurable this tick (container already exited, or `du` unreadable). Enforcing a + // quota on a number we do not have would be wrong, and reporting it as 0 would wipe the + // last known figure off the snapshot — so report "unknown" and let the job continue. + CORE_LOGGER.debug( + `[metrics] job ${job.jobId}: disk usage unmeasurable this tick — keeping the last known figure` + ) + return { canContinue: true } + } const baseImageSize = this.jobImageSizes.get(job.jobId) || 0 const algorithmUsage = Math.max(0, totalUsage - baseImageSize) @@ -2907,14 +2985,217 @@ export class C2DEngineDocker extends C2DEngine { job.algoStopTimestamp = String(Date.now() / 1000) job.dateFinished = String(Date.now() / 1000) + // Final metrics snapshot before the container is torn down (records the last CPU + // seconds, peak memory, final disk usage and exit info for postmortems). + await this.collectJobMetrics(job, algorithmUsage, true) await this.db.updateJob(job) await this.cleanupJob(job) CORE_LOGGER.info(`Job ${job.jobId} terminated - DISK QUOTA EXCEEDED`) - return false + return { canContinue: false, usedBytes: algorithmUsage } } - return true + return { canContinue: true, usedBytes: algorithmUsage } + } + + // Resolves the container's requested allocation as bytes/cores for the metrics snapshot. + private getJobMetricsAllocation(job: DBComputeJob): { + cpu: number + ramBytes: number + diskBytes: number + } { + const cpu = this.getResourceRequest(job.resources, 'cpu') || 0 + const ramGb = this.getResourceRequest(job.resources, 'ram') || 0 + const diskGb = this.getDiskQuota(job) + return { + cpu, + ramBytes: ramGb * 1024 * 1024 * 1024, + diskBytes: diskGb * 1024 * 1024 * 1024 + } + } + + // ONE line per sampling interval with the engine's whole live resource picture, plus a + // "pressure" line for each workload that is close to a limit. This is the admin's entry + // point into the metrics: `grep '\[metrics\]'` for everything, `grep '\[metrics\] summary'` + // for the roll-up, `grep '\[metrics\] pressure'` for what is about to hurt. + // + // Reads the snapshots already sampled this tick (no extra Docker or DB calls) and is + // throttled to C2D_METRICS_INTERVAL_SECONDS — the loop itself ticks every 2s, but snapshots + // only refresh once per interval, so logging every tick would just repeat numbers. + // Never throws: a logging failure must not touch the loop. + private logMetricsSummary( + jobs: DBComputeJob[] = [], + services: ServiceJob[] = [] + ): void { + try { + if (!isMetricsCollectionEnabled()) return + const now = Date.now() + if (now - (this.lastMetricsSummaryAt ?? 0) < getMetricsIntervalSeconds() * 1000) { + return + } + this.lastMetricsSummaryAt = now + + const sampled: Array<{ + kind: string + id: string + metrics: ContainerMetricsSnapshot + }> = [] + for (const job of jobs) { + if (job.runtimeMetrics) { + sampled.push({ kind: 'job', id: job.jobId, metrics: job.runtimeMetrics }) + } + } + for (const svc of services) { + if (svc.runtimeMetrics) { + sampled.push({ + kind: 'service', + id: svc.serviceId, + metrics: svc.runtimeMetrics + }) + } + } + + if (sampled.length === 0) { + // Nothing running, or nothing sampled yet — say which, so silence is never ambiguous. + if (jobs.length + services.length > 0) { + CORE_LOGGER.debug( + `[metrics] summary engine ${this.getC2DConfig().hash}: ${jobs.length} job(s) / ` + + `${services.length} service(s) running, none sampled yet` + ) + } + return + } + + let cpuPercent = 0 + let coresAllocated = 0 + let memUsed = 0 + let memLimit = 0 + let diskUsed = 0 + let rxBytes = 0 + let txBytes = 0 + let gpuDevices = 0 + let throttledCount = 0 + let oldestSampleAgeSeconds = 0 + for (const { metrics } of sampled) { + cpuPercent += metrics.cpu.usagePercent + coresAllocated += metrics.cpu.allocated + memUsed += metrics.memory.usageBytes + memLimit += metrics.memory.limitBytes + diskUsed += metrics.disk.usedBytes + rxBytes += metrics.network?.rxBytes ?? 0 + txBytes += metrics.network?.txBytes ?? 0 + gpuDevices += metrics.gpu?.length ?? 0 + if (metrics.cpu.throttledPeriods > 0) throttledCount++ + const age = (now - new Date(metrics.collectedAt).getTime()) / 1000 + if (Number.isFinite(age) && age > oldestSampleAgeSeconds) { + oldestSampleAgeSeconds = age + } + } + const hostCores = this.physicalLimits.get('cpu') ?? 0 + + CORE_LOGGER.debug( + `[metrics] summary engine ${this.getC2DConfig().hash}: ` + + `${jobs.length} job(s) / ${services.length} service(s), ${sampled.length} sampled | ` + + `cpu ${cpuPercent.toFixed(1)}% of host${hostCores ? ` (${hostCores} core(s))` : ''}, ` + + `${coresAllocated} core(s) allocated, ${throttledCount} throttled | ` + + `mem ${formatBytes(memUsed)}/${formatBytes(memLimit)} allocated | ` + + `disk ${formatBytes(diskUsed)} | net rx ${formatBytes(rxBytes)} tx ${formatBytes(txBytes)} | ` + + `gpu ${gpuDevices} device(s) | oldest sample ${oldestSampleAgeSeconds.toFixed(0)}s ago` + ) + + for (const { kind, id, metrics } of sampled) { + const pressure: string[] = [] + if (metrics.memory.limitBytes > 0 && metrics.memory.usagePercent >= 90) { + pressure.push( + `mem ${metrics.memory.usagePercent}% of limit (OOM kill risk, peak ` + + `${formatBytes(metrics.memory.peakUsageBytes)})` + ) + } + if (metrics.disk.usagePercent !== undefined && metrics.disk.usagePercent >= 90) { + pressure.push(`disk ${metrics.disk.usagePercent}% of quota (job stop risk)`) + } + if (metrics.pids.limit > 0 && metrics.pids.current / metrics.pids.limit >= 0.8) { + pressure.push(`pids ${metrics.pids.current}/${metrics.pids.limit}`) + } + if (metrics.cpu.throttledPeriods > 0) { + pressure.push( + `cpu throttled ${metrics.cpu.throttledPeriods} periods / ` + + `${metrics.cpu.throttledSeconds}s (undersized cpu request)` + ) + } + if (pressure.length > 0) { + CORE_LOGGER.debug(`[metrics] pressure ${kind} ${id}: ${pressure.join(', ')}`) + } + } + } catch (e: any) { + CORE_LOGGER.debug(`[metrics] summary failed: ${e?.message}`) + } + } + + // Best-effort: samples the running algorithm container (+ its GPUs) and stores the snapshot + // on job.runtimeMetrics. NEVER throws — a metrics failure must not touch the state machine. + // Does not persist by itself; callers let their existing db.updateJob() write it. Reuses the + // du(/) result from monitorDiskUsage. `force` writes a final snapshot regardless of staleness. + private async collectJobMetrics( + job: DBComputeJob, + diskUsedBytes?: number, + force: boolean = false + ): Promise { + try { + if (!isMetricsCollectionEnabled()) return false + if (!force && !isSnapshotStale(job.runtimeMetrics, getMetricsIntervalSeconds())) { + return false + } + const firstSample = !job.runtimeMetrics + const raw = await sampleContainerMetrics(this.docker, job.jobId + '-algoritm') + if (!raw) return false + const snap = buildSnapshot( + raw, + job.runtimeMetrics, + this.getJobMetricsAllocation(job), + diskUsedBytes + ) + const env = this.envs.find((e) => e.id === job.environment) + const gpu = await this.gpuMetrics.collect(job.resources, env?.resources ?? []) + if (gpu) snap.gpu = gpu + job.runtimeMetrics = snap + CORE_LOGGER.debug( + `[metrics] job ${job.jobId}${force ? ' [final]' : ''}: ${describeSnapshot(snap)}` + ) + if (firstSample) { + // One-shot stats carry no previous CPU counters, so the very first sample cannot + // compute a rate — worth saying out loud, since "cpu 0%" right after a job starts + // looks like broken collection rather than a missing baseline. + // + // Seeing this repeatedly for the SAME job is a different story: the snapshot is not + // surviving the round-trip to the DB, so CPU % can never be computed and the owner's + // status response will never carry runtimeMetrics. Call that out explicitly instead of + // leaving an admin to notice that "first snapshot" keeps reappearing. + const runtimeSeconds = + Date.now() / 1000 - parseFloat(job.algoStartTimestamp || '0') + if ( + Number.isFinite(runtimeSeconds) && + runtimeSeconds > getMetricsIntervalSeconds() * 2 + ) { + CORE_LOGGER.warn( + `[metrics] job ${job.jobId}: no previous snapshot after ${runtimeSeconds.toFixed( + 0 + )}s of runtime — the snapshot is not being persisted, so cpu usagePercent stays 0 ` + + 'and the owner status response carries no runtimeMetrics. Check that this node runs ' + + 'a build where runtimeMetrics is part of the c2djobs body blob.' + ) + } else { + CORE_LOGGER.debug( + `[metrics] job ${job.jobId}: first snapshot — cpu usagePercent stays 0 until the next ` + + `sample (~${getMetricsIntervalSeconds()}s), it needs two samples to compute a delta` + ) + } + } + return true + } catch (e: any) { + CORE_LOGGER.debug(`[metrics] job ${job.jobId}: collection failed: ${e?.message}`) + return false + } } private async pullImage(originaljob: DBComputeJob) { @@ -3813,7 +4094,9 @@ export class C2DEngineDocker extends C2DEngine { // Checked every InternalLoop tick (same cadence as compute jobs) so a service whose // container died on its own (crash, OOM, or the whole Docker daemon going down) is // detected within ~cronTime instead of only at expiresAt. - private async checkRunningServices(): Promise { + // Returns the services it checked (health + metrics sampled), so the caller can roll their + // snapshots into the engine-wide metrics summary without re-querying. + private async checkRunningServices(): Promise { const services = await this.db.getRunningServiceJobs(this.getC2DConfig().hash) // Skip services with a lifecycle op in flight: a restart intentionally kills the // container mid-way, which must not be reported as an unexpected death. @@ -3823,6 +4106,7 @@ export class C2DEngineDocker extends C2DEngine { !this.serviceOpsInFlight.has(svc.serviceId) ) await Promise.all(runningOnly.map((svc) => this.checkServiceContainerHealth(svc))) + return runningOnly } private async checkServiceContainerHealth(job: ServiceJob): Promise { @@ -3840,7 +4124,150 @@ export class C2DEngineDocker extends C2DEngine { : details.State.Error ? `error: ${details.State.Error}` : `exited with code ${details.State.ExitCode}` - await this.markServiceFailed(job, reason) + await this.markServiceFailed(job, reason, details.State) + return + } + // Container healthy: sample live runtime metrics (throttled, best-effort, lease-free). + await this.sampleAndPersistServiceMetrics(job) + } + + // Resolves a service's requested allocation as bytes/cores for the metrics snapshot. + // Services have no /data volume, so disk usage comes from the container writable layer + // (SizeRw) rather than a quota — diskBytes is left 0 (no quota to report a % against). + private getServiceMetricsAllocation(job: ServiceJob): { + cpu: number + ramBytes: number + diskBytes: number + } { + const cpu = job.resources?.find((r) => r.id === 'cpu')?.amount || 0 + const ramGb = job.resources?.find((r) => r.id === 'ram')?.amount || 0 + return { cpu, ramBytes: ramGb * 1024 * 1024 * 1024, diskBytes: 0 } + } + + // Samples a running service container (+ its GPUs) and persists the snapshot. Throttled to + // C2D_METRICS_INTERVAL_SECONDS and strictly best-effort. + // + // Deliberately does NOT take the service lifecycle lease: that lease is user-facing (stop / + // restart / extend throw "operation in progress" when it is held), so acquiring it for a + // metrics sample could make a real lifecycle operation fail. Instead it: (1) skips when a + // lifecycle op is in flight locally or cross-process (the shared DB lease), and (2) persists + // via a guarded, metrics-ONLY write that leaves lifecycle fields untouched and no-ops unless the + // row still matches the sampled owner/clusterHash/status/containerId. So a metrics write can + // never block, fail, or clobber a lifecycle transition — even across processes sharing the DB. + private async sampleAndPersistServiceMetrics(job: ServiceJob): Promise { + try { + if (!isMetricsCollectionEnabled()) return + if (!isSnapshotStale(job.runtimeMetrics, getMetricsIntervalSeconds())) return + if (this.serviceOpsInFlight.has(job.serviceId)) return + const raw = await sampleContainerMetrics(this.docker, job.containerId, { + size: true + }) + if (!raw) return + const [fresh] = await this.db.getServiceJob(job.serviceId, job.owner) + if ( + !fresh || + fresh.status !== ServiceStatusNumber.Running || + fresh.containerId !== job.containerId + ) { + CORE_LOGGER.debug( + `[metrics] service ${job.serviceId}: dropping sample, the record moved on ` + + `(status ${fresh?.status ?? 'gone'}, container ${ + fresh?.containerId ?? 'gone' + } vs sampled ${job.containerId})` + ) + return + } + const snap = buildSnapshot( + raw, + fresh.runtimeMetrics, + this.getServiceMetricsAllocation(fresh), + undefined + ) + const connResources: ComputeResource[] = + this.getC2DConfig().connection?.resources ?? [] + const gpu = await this.gpuMetrics.collect(fresh.resources, connResources) + if (gpu) snap.gpu = gpu + // Skip if a lifecycle op is in flight — locally (this process) or cross-process (the shared + // DB lease). A metrics sample must never race a stop/restart/extend. + if ( + this.serviceOpsInFlight.has(job.serviceId) || + (await this.db.isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS)) + ) { + CORE_LOGGER.debug( + `[metrics] service ${job.serviceId}: dropping sample, a lifecycle operation is in flight` + ) + return + } + // Persist ONLY runtimeMetrics, guarded on the unchanged serviceId + owner/clusterHash/ + // status/containerId, so we never overwrite a lifecycle transition made by another process. + const written = await this.db.updateServiceJobMetrics( + job.serviceId, + { + owner: fresh.owner, + clusterHash: fresh.clusterHash, + status: ServiceStatusNumber.Running, + containerId: fresh.containerId + }, + snap + ) + CORE_LOGGER.debug( + `[metrics] service ${job.serviceId} (${written ? 'persisted' : 'guarded write no-op'}): ` + + describeSnapshot(snap) + ) + } catch (e: any) { + CORE_LOGGER.debug( + `[metrics] service ${job.serviceId}: collection failed: ${e?.message}` + ) + } + } + + // Stamps a dead container's exit info (status / exitCode / OOM / restart count) onto the + // job's LAST snapshot without overwriting the live cpu/memory/peak figures already captured + // while it ran. Used by markServiceFailed so the structured containerState survives (today + // the reason lives only in free-text statusText). Returns the previous snapshot unchanged + // when no state is available. + private stampServiceExitInfo( + prev: ContainerMetricsSnapshot | undefined, + state: any + ): ContainerMetricsSnapshot | undefined { + if (!state) return prev + const base: ContainerMetricsSnapshot = + prev ?? + ({ + collectedAt: new Date().toISOString(), + containerState: { status: 'exited', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 0, + allocated: 0, + usagePercentOfAllocated: 0, + cumulativeSeconds: 0, + throttledPeriods: 0, + throttledSeconds: 0 + }, + memory: { usageBytes: 0, limitBytes: 0, usagePercent: 0, peakUsageBytes: 0 }, + disk: { usedBytes: 0 }, + blockIO: { readBytes: 0, writeBytes: 0 }, + pids: { current: 0, limit: 512 } + } as ContainerMetricsSnapshot) + return { + ...base, + collectedAt: new Date().toISOString(), + containerState: { + status: String(state.Status ?? 'exited'), + startedAt: state.StartedAt ?? base.containerState.startedAt, + finishedAt: + state.FinishedAt && !String(state.FinishedAt).startsWith('0001-01-01') + ? state.FinishedAt + : base.containerState.finishedAt, + exitCode: state.ExitCode, + oomKilled: Boolean(state.OOMKilled), + error: state.Error || undefined, + restartCount: + typeof state.RestartCount === 'number' + ? state.RestartCount + : base.containerState.restartCount, + health: state.Health?.Status ?? base.containerState.health + } } } @@ -3848,7 +4275,11 @@ export class C2DEngineDocker extends C2DEngine { // touch Docker or release host ports/network — the consumer already paid for those and // restartService() reuses them (and does its own best-effort teardown of the dead // container/network first). - private async markServiceFailed(job: ServiceJob, reason: string): Promise { + private async markServiceFailed( + job: ServiceJob, + reason: string, + exitState?: any + ): Promise { // Take the SAME lifecycle lease every start/stop/restart holds, so the Error write // is serialized with them — a check-then-write here could otherwise overwrite a // Restarting state persisted between our lease check and our update. Failing to @@ -3874,6 +4305,10 @@ export class C2DEngineDocker extends C2DEngine { } fresh.status = ServiceStatusNumber.Error fresh.statusText = `service container exited unexpectedly: ${reason}` + // Preserve the last live metrics and stamp the structured exit info onto them. + if (isMetricsCollectionEnabled()) { + fresh.runtimeMetrics = this.stampServiceExitInfo(fresh.runtimeMetrics, exitState) + } await this.db.updateServiceJob(fresh) CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) } finally { @@ -3881,6 +4316,34 @@ export class C2DEngineDocker extends C2DEngine { } } + // Best-effort final metrics snapshot of a still-running service container, taken before a + // stop tears it down. Must be called while holding the service lifecycle lock; does not + // persist — the caller's own updateServiceJob() writes job.runtimeMetrics. Never throws. + private async captureFinalServiceSnapshot(job: ServiceJob): Promise { + try { + if (!isMetricsCollectionEnabled() || !job.containerId) return + const raw = await sampleContainerMetrics(this.docker, job.containerId, { + size: true + }) + if (!raw) return + const snap = buildSnapshot( + raw, + job.runtimeMetrics, + this.getServiceMetricsAllocation(job), + undefined + ) + const connResources: ComputeResource[] = + this.getC2DConfig().connection?.resources ?? [] + const gpu = await this.gpuMetrics.collect(job.resources, connResources) + if (gpu) snap.gpu = gpu + job.runtimeMetrics = snap + } catch (e: any) { + CORE_LOGGER.debug( + `captureFinalServiceSnapshot failed for ${job.serviceId}: ${e?.message}` + ) + } + } + // Tries to take the per-service lifecycle lock: the in-memory set serializes callers // inside this process, the service_locks DB lease serializes across processes sharing // the DB + Docker daemon. Returns false when someone else owns the service. @@ -4034,6 +4497,8 @@ export class C2DEngineDocker extends C2DEngine { `containerId=${job.containerId || '-'}, networkId=${job.networkId || '-'})` ) await this.logServiceDockerState(`stop ${serviceId}: before teardown`, serviceId) + // Final runtime-metrics snapshot while the container is still up (best-effort). + await this.captureFinalServiceSnapshot(job) job.status = ServiceStatusNumber.Stopping job.statusText = ServiceStatusText[ServiceStatusNumber.Stopping] await this.db.updateServiceJob(job) @@ -4204,6 +4669,8 @@ export class C2DEngineDocker extends C2DEngine { // claimTx is set, so recovery never touches escrow for a restart). if (job.containerId) { CORE_LOGGER.debug(`restart ${serviceId}: removing old container ${job.containerId}`) + // Final snapshot of the outgoing container before it is torn down (best-effort). + await this.captureFinalServiceSnapshot(job) const c = this.docker.getContainer(job.containerId) await c.stop({ t: 10 }).catch((e) => { CORE_LOGGER.debug(`restart ${serviceId}: old container stop: ${e.message}`) @@ -4219,6 +4686,9 @@ export class C2DEngineDocker extends C2DEngine { job.containerId = '' job.networkId = '' + // Reset the metrics accumulators: the new container is a fresh process, so peak memory + // and CPU deltas must not carry over from the outgoing one. + job.runtimeMetrics = undefined await this.db.updateServiceJob(job) // Live Docker handles for the newly-created container/network, tracked so the diff --git a/src/components/c2d/containerMetrics.ts b/src/components/c2d/containerMetrics.ts new file mode 100644 index 000000000..7760fe43c --- /dev/null +++ b/src/components/c2d/containerMetrics.ts @@ -0,0 +1,317 @@ +import type Dockerode from 'dockerode' +import type { ContainerMetricsSnapshot } from '../../@types/C2D/C2D.js' +import { CORE_LOGGER } from '../../utils/logging/common.js' +import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js' + +// PidsLimit applied to every job/service container at creation time. +const CONTAINER_PIDS_LIMIT = 512 +const DEFAULT_METRICS_INTERVAL_SECONDS = 10 + +// Raw material for a snapshot: the Docker stats blob + inspect State, kept loosely typed +// because the shape drifts between cgroup v1/v2 and Docker API versions. buildSnapshot() +// reads it defensively. +export interface RawContainerSample { + stats: any + state: any // Dockerode.ContainerInspectInfo['State'] +} + +// Sampling cadence (seconds). Default 10; `0` disables collection entirely. Parsed from +// C2D_METRICS_INTERVAL_SECONDS; invalid/negative values fall back to the default (use 0 to +// disable). Returns seconds — callers compare against snapshot age. +export function getMetricsIntervalSeconds(): number { + const raw = ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value + if (raw === undefined || raw === null || raw === '') { + return DEFAULT_METRICS_INTERVAL_SECONDS + } + const parsed = parseInt(raw, 10) + if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_METRICS_INTERVAL_SECONDS + return parsed +} + +export function isMetricsCollectionEnabled(): boolean { + return getMetricsIntervalSeconds() > 0 +} + +// Compact byte formatting for log lines ("1.4 GiB", "512 B") — raw byte counts are unreadable +// when scanning metrics debug output. +export function formatBytes(bytes: number): string { + const value = toNum(bytes) + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'] + let idx = 0 + let scaled = value + while (scaled >= 1024 && idx < units.length - 1) { + scaled /= 1024 + idx++ + } + return `${idx === 0 ? scaled : scaled.toFixed(1)} ${units[idx]}` +} + +// One-line, human-scannable rendering of a snapshot for CORE_LOGGER.debug — the whole +// `docker stats` view of the container plus what the CLI does not show (throttling, peak +// memory, disk vs quota, GPU). Keep it single-line so operators can grep a job id and read +// the series of samples top to bottom. +export function describeSnapshot(snapshot: ContainerMetricsSnapshot): string { + const { cpu, memory, disk, pids, network, blockIO, containerState, gpu } = snapshot + const parts = [ + `cpu ${cpu.usagePercent}% (${cpu.usagePercentOfAllocated}% of ${cpu.allocated} core(s), ` + + `${cpu.cumulativeSeconds}s used, throttled ${cpu.throttledPeriods} periods/${cpu.throttledSeconds}s)`, + `mem ${formatBytes(memory.usageBytes)}/${formatBytes(memory.limitBytes)} ` + + `(${memory.usagePercent}%, peak ${formatBytes(memory.peakUsageBytes)})`, + `disk ${formatBytes(disk.usedBytes)}${ + disk.quotaBytes ? `/${formatBytes(disk.quotaBytes)} (${disk.usagePercent}%)` : '' + }`, + `pids ${pids.current}/${pids.limit}`, + `net rx ${network ? formatBytes(network.rxBytes) : 'n/a'} tx ${ + network ? formatBytes(network.txBytes) : 'n/a' + }`, + `blkio r ${formatBytes(blockIO.readBytes)} w ${formatBytes(blockIO.writeBytes)}`, + // Docker reports ExitCode 0 on a RUNNING container, so only show it once the container is + // actually finished — otherwise the line reads "state running exit=0", which looks like a + // container that both is and is not running. + `state ${containerState.status}${containerState.oomKilled ? ' OOMKilled' : ''}${ + containerState.status !== 'running' && + containerState.exitCode !== undefined && + containerState.exitCode !== null + ? ` exit=${containerState.exitCode}` + : '' + }${containerState.health ? ` health=${containerState.health}` : ''}` + ] + if (gpu?.length) { + parts.push( + `gpu ${gpu + .map( + (g) => + `${g.resourceId}=${g.utilizationPercent ?? 'n/a'}%/${ + g.memoryUsedBytes !== null && g.memoryUsedBytes !== undefined + ? formatBytes(g.memoryUsedBytes) + : 'n/a' + }` + ) + .join(' ')}` + ) + } + return parts.join(', ') +} + +// True when the previous snapshot is older than the sampling interval (or there is none). +// Kept a pure helper so the engine wiring and unit tests share one staleness rule. `now` +// is injectable for deterministic tests. +export function isSnapshotStale( + prev: ContainerMetricsSnapshot | undefined, + intervalSeconds: number, + now: number = Date.now() +): boolean { + if (!prev || !prev.collectedAt) return true + const age = (now - new Date(prev.collectedAt).getTime()) / 1000 + return age >= intervalSeconds +} + +// Wraps a container's stats + inspect into a RawContainerSample. Never throws into the loop: +// on any failure (container gone mid-sample, daemon hiccup) it logs at debug and returns +// null so the caller keeps the previous snapshot. Uses one-shot stats (no daemon-side ~1s +// double sample) — deltas are computed by buildSnapshot from the previous snapshot instead. +export async function sampleContainerMetrics( + docker: Dockerode, + containerId: string, + opts: { size?: boolean } = {} +): Promise { + try { + const container = docker.getContainer(containerId) + // one-shot returns instantly without precpu_stats; we self-compute CPU deltas. + const stats: any = await container.stats({ stream: false, 'one-shot': true } as any) + // Cast around the dockerode overloads: `size` is a valid query param but not modeled on + // every @types/dockerode version. + const info: any = await (container.inspect as any)( + opts.size ? { size: true } : undefined + ) + return { stats, state: { ...info.State, SizeRw: info.SizeRw } } + } catch (e: any) { + CORE_LOGGER.debug( + `[metrics] could not sample container ${containerId}: ${e?.message}` + ) + return null + } +} + +function toNum(v: any): number { + return typeof v === 'number' && Number.isFinite(v) ? v : 0 +} + +function sumBlkio(entries: any[], op: 'Read' | 'Write'): number { + if (!Array.isArray(entries)) return 0 + return entries + .filter((e) => e && String(e.op).toLowerCase() === op.toLowerCase()) + .reduce((acc, e) => acc + toNum(e.value), 0) +} + +function sumNetworks(networks: any): { rxBytes: number; txBytes: number } | undefined { + if (!networks || typeof networks !== 'object') return undefined + let rx = 0 + let tx = 0 + for (const iface of Object.values(networks)) { + rx += toNum(iface?.rx_bytes) + tx += toNum(iface?.tx_bytes) + } + return { rxBytes: rx, txBytes: tx } +} + +// Pure transform: raw sample + previous snapshot → a new snapshot. Handles cgroup v1/v2 +// field drift (defaults missing fields to 0), self-computes CPU % from the previous +// sample's cumulative counters (one-shot stats have no precpu_stats), tracks the memory +// peak across samples, and carries the `prev` accumulator for the next delta. Never throws; +// every read is defensive. Easily unit-tested with fixture JSON. +export function buildSnapshot( + raw: RawContainerSample, + prev: ContainerMetricsSnapshot | undefined, + alloc: { cpu: number; ramBytes: number; diskBytes: number }, + diskUsedBytes: number | undefined, + now: number = Date.now() +): ContainerMetricsSnapshot { + const stats = raw?.stats ?? {} + const state = raw?.state ?? {} + const collectedAt = new Date(now).toISOString() + + // ---- CPU ---- + const cpuStats = stats.cpu_stats ?? {} + const cpuUsage = cpuStats.cpu_usage ?? {} + const totalUsage = toNum(cpuUsage.total_usage) // ns, monotonic + const systemCpu = toNum(cpuStats.system_cpu_usage) // ns + const onlineCpus = + toNum(cpuStats.online_cpus) || + (Array.isArray(cpuUsage.percpu_usage) ? cpuUsage.percpu_usage.length : 0) + + // CPU % needs a previous sample. Prefer precpu_stats (non-one-shot); otherwise our stored + // accumulator. On a true FIRST sample (neither present) we cannot compute a rate — report + // 0 rather than a meaningless cumulative-since-boot ratio. + const preCpu = stats.precpu_stats ?? {} + const hasPreCpu = + toNum(preCpu?.cpu_usage?.total_usage) > 0 && toNum(preCpu?.system_cpu_usage) > 0 + const hasPrevAccumulator = + toNum(prev?.prev?.cpuTotal) > 0 && toNum(prev?.prev?.systemCpu) > 0 + const preTotal = hasPreCpu + ? toNum(preCpu.cpu_usage.total_usage) + : toNum(prev?.prev?.cpuTotal) + const preSystem = hasPreCpu + ? toNum(preCpu.system_cpu_usage) + : toNum(prev?.prev?.systemCpu) + + const cpuDelta = totalUsage - preTotal + const systemDelta = systemCpu - preSystem + let usagePercent = 0 + if ( + (hasPreCpu || hasPrevAccumulator) && + cpuDelta > 0 && + systemDelta > 0 && + onlineCpus > 0 + ) { + usagePercent = (cpuDelta / systemDelta) * onlineCpus * 100 + } + usagePercent = Math.max(0, Number(usagePercent.toFixed(2))) + + // Cumulative counters can only grow. A container that has already exited reports zeros + // (its cgroup is gone), which must NOT wipe what it consumed while it ran — the final + // snapshot is exactly the one a postmortem reads. So every monotonic figure keeps the + // highest value seen. `monotonic()` also covers a daemon returning a partial stats blob. + const monotonic = (current: number, previous: number | undefined): number => + Math.max(toNum(current), toNum(previous)) + + const throttling = cpuStats.throttling_data ?? {} + const cpu = { + usagePercent, + allocated: alloc.cpu, + usagePercentOfAllocated: + alloc.cpu > 0 ? Number((usagePercent / alloc.cpu).toFixed(2)) : 0, + cumulativeSeconds: monotonic( + Number((totalUsage / 1e9).toFixed(3)), + prev?.cpu?.cumulativeSeconds + ), + throttledPeriods: monotonic( + toNum(throttling.throttled_periods), + prev?.cpu?.throttledPeriods + ), + throttledSeconds: monotonic( + Number((toNum(throttling.throttled_time) / 1e9).toFixed(3)), + prev?.cpu?.throttledSeconds + ) + } + + // ---- Memory ---- + const memStats = stats.memory_stats ?? {} + const inactiveFile = toNum(memStats.stats?.inactive_file ?? memStats.stats?.cache) + const usageBytes = Math.max(0, toNum(memStats.usage) - inactiveFile) + const limitBytes = alloc.ramBytes > 0 ? alloc.ramBytes : toNum(memStats.limit) + const peakUsageBytes = Math.max(usageBytes, toNum(prev?.memory?.peakUsageBytes)) + const memory = { + usageBytes, + limitBytes, + usagePercent: + limitBytes > 0 ? Number(((usageBytes / limitBytes) * 100).toFixed(2)) : 0, + peakUsageBytes + } + + // ---- Disk ---- + // Order of preference: the caller's fresh measurement (jobs: `du` minus base image) → + // the writable-layer size from inspect (services) → the last known figure. Disk is a gauge, + // not a counter, but "unmeasurable" (the container is gone) must not read as "0 bytes used". + const measuredDisk = + diskUsedBytes !== undefined && diskUsedBytes !== null + ? diskUsedBytes + : toNum(state.SizeRw) + const resolvedDisk = + measuredDisk > 0 ? measuredDisk : toNum(prev?.disk?.usedBytes) || measuredDisk + const disk: ContainerMetricsSnapshot['disk'] = { usedBytes: Math.max(0, resolvedDisk) } + if (alloc.diskBytes > 0) { + disk.quotaBytes = alloc.diskBytes + disk.usagePercent = Number(((disk.usedBytes / alloc.diskBytes) * 100).toFixed(2)) + } + + // ---- Network / Block IO / PIDs ---- + // Network and block I/O are cumulative byte counters, so they get the same monotonic + // treatment as CPU seconds: an exited container reports nothing, and losing the totals it + // transferred would defeat the point of the final snapshot. + const sampledNetwork = sumNetworks(stats.networks) + const network = + sampledNetwork || prev?.network + ? { + rxBytes: monotonic(sampledNetwork?.rxBytes ?? 0, prev?.network?.rxBytes), + txBytes: monotonic(sampledNetwork?.txBytes ?? 0, prev?.network?.txBytes) + } + : undefined + const blkio = stats.blkio_stats?.io_service_bytes_recursive + const blockIO = { + readBytes: monotonic(sumBlkio(blkio, 'Read'), prev?.blockIO?.readBytes), + writeBytes: monotonic(sumBlkio(blkio, 'Write'), prev?.blockIO?.writeBytes) + } + const pids = { + current: toNum(stats.pids_stats?.current), + limit: toNum(stats.pids_stats?.limit) || CONTAINER_PIDS_LIMIT + } + + const snapshot: ContainerMetricsSnapshot = { + collectedAt, + containerState: { + status: String(state.Status ?? (state.Running ? 'running' : 'unknown')), + startedAt: state.StartedAt, + finishedAt: + state.FinishedAt && !String(state.FinishedAt).startsWith('0001-01-01') + ? state.FinishedAt + : undefined, + exitCode: state.ExitCode, + oomKilled: Boolean(state.OOMKilled), + error: state.Error || undefined, + restartCount: toNum(state.RestartCount), + health: state.Health?.Status + }, + cpu, + memory, + disk, + blockIO, + pids, + prev: { cpuTotal: totalUsage, systemCpu, sampledAt: collectedAt } + } + if (network) snapshot.network = network + // GPU is attached by the caller (GpuMetricsService) after this pure transform, since it + // needs host-side, job-resource-driven resolution outside the Docker stats blob. + if (prev?.gpu) snapshot.gpu = prev.gpu + return snapshot +} diff --git a/src/components/c2d/gpu/index.ts b/src/components/c2d/gpu/index.ts new file mode 100644 index 000000000..3d0570ae8 --- /dev/null +++ b/src/components/c2d/gpu/index.ts @@ -0,0 +1,150 @@ +import type { + ComputeResource, + ComputeResourceRequest, + GpuMetricsSnapshot +} from '../../../@types/C2D/C2D.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { GpuDeviceHandle, GpuVendor, GpuVendorCollector } from './types.js' +import { NvmlGpuCollector } from './nvml.js' + +export { parseMemoryTotalToBytes } from './types.js' + +// Reads GPU_METRICS: "auto" (default) enables detect-and-pick per vendor; "off" disables +// GPU collection entirely. A per-vendor override JSON is accepted for forward-compat but, +// with only the NVIDIA backend implemented, currently just gates whether nvidia is on. +function gpuMetricsEnabled(): boolean { + const raw = (ENVIRONMENT_VARIABLES.GPU_METRICS.value ?? 'auto').trim().toLowerCase() + return raw !== 'off' && raw !== 'false' && raw !== '0' +} + +// Infers a GPU resource's vendor when `platform` was omitted. Only NVIDIA can be inferred +// cheaply here (deviceRequests.Driver === 'nvidia'); AMD/Intel need PCI/DRM inspection that +// their (deferred) backends will own. Returns null when the vendor is unknown/unsupported. +function inferVendor(res: ComputeResource): GpuVendor | null { + const platform = String(res.platform ?? '').toLowerCase() + if (platform === 'nvidia' || platform === 'amd' || platform === 'intel') { + return platform as GpuVendor + } + if (String(res.init?.deviceRequests?.Driver ?? '').toLowerCase() === 'nvidia') { + return 'nvidia' + } + return null +} + +// Resolves the GPUs a job/service actually holds and samples each one, attaching a per-device +// entry to the container snapshot. Driven entirely by job.resources → the env resource pool, +// so pure-CPU jobs never touch any GPU code. Best-effort: any failure yields no `gpu` field +// rather than disturbing the metrics/state loop. NVIDIA only today (NVML); AMD/Intel resources +// are skipped with a one-time debug note until their backends land. +export class GpuMetricsService { + private collectors: Map = new Map() + private unsupportedWarned = new Set() + // Resource ids whose vendor could not be determined — logged once each, not every sample. + private unresolvedWarned = new Set() + + private getCollector(vendor: GpuVendor): GpuVendorCollector | null { + if (this.collectors.has(vendor)) return this.collectors.get(vendor)! + let collector: GpuVendorCollector | null = null + if (vendor === 'nvidia') collector = new NvmlGpuCollector() + // AMD/Intel backends are deferred; their resources are skipped (see below). + if (collector) this.collectors.set(vendor, collector) + return collector + } + + // Returns undefined (not []) when there is nothing to report, so the caller can leave the + // snapshot's `gpu` field absent for CPU-only jobs / disabled collection. + async collect( + jobResources: ComputeResourceRequest[], + envResources: ComputeResource[] + ): Promise { + if (!gpuMetricsEnabled() || !jobResources?.length || !envResources?.length) { + return undefined + } + + // Group the held GPU resources by vendor so each backend samples its devices in one sweep. + const byVendor = new Map() + let gpusHeld = 0 + try { + for (const req of jobResources) { + if (!req || req.amount <= 0) continue + const res = envResources.find((r) => r.id === req.id) + if (!res || String(res.type).toLowerCase() !== 'gpu') continue + gpusHeld++ + const vendor = inferVendor(res) + if (!vendor) { + // The single most likely reason an operator sees no GPU numbers: the resource does + // not say which vendor it is. Name the resource and the fix, once per resource. + if (!this.unresolvedWarned.has(res.id)) { + this.unresolvedWarned.add(res.id) + CORE_LOGGER.debug( + `[metrics] gpu: cannot determine the vendor of resource "${res.id}" — it has no ` + + '"platform" ("nvidia" | "amd" | "intel") and no init.deviceRequests.Driver. Set ' + + '"platform" on that resource in DOCKER_COMPUTE_ENVIRONMENTS to get GPU metrics.' + ) + } + continue + } + const collector = this.getCollector(vendor) + if (!collector) { + if (!this.unsupportedWarned.has(vendor)) { + this.unsupportedWarned.add(vendor) + CORE_LOGGER.debug( + `[metrics] gpu: ${vendor} backend not implemented yet — skipping ${res.id}` + ) + } + continue + } + const handle = collector.resolve(res) + if (!handle) continue + const list = byVendor.get(vendor) ?? [] + list.push(handle) + byVendor.set(vendor, list) + } + + const out: GpuMetricsSnapshot[] = [] + for (const [vendor, handles] of byVendor.entries()) { + const collector = this.getCollector(vendor) + if (!collector) continue + // eslint-disable-next-line no-await-in-loop + const metrics = await collector.sample(handles) + for (const m of metrics) { + out.push({ + resourceId: m.resourceId, + vendor: m.vendor, + utilizationPercent: m.utilizationPercent, + memoryUsedBytes: m.memoryUsedBytes, + memoryTotalBytes: m.memoryTotalBytes, + temperatureC: m.temperatureC, + powerWatts: m.powerWatts, + shared: m.shared + }) + } + } + if (gpusHeld > 0 && !out.length) { + // The workload holds GPUs but nothing came back — the snapshot will simply have no + // gpu[]. Usually a missing/unreadable vendor backend on the host, or a resource whose + // `platform` could not be inferred; say so instead of silently omitting the field. + CORE_LOGGER.debug( + `[metrics] gpu: ${gpusHeld} GPU resource(s) held but no device metrics resolved — ` + + 'snapshot carries no gpu[] (check the resource platform and the host driver/tooling)' + ) + } + return out.length ? out : undefined + } catch (e: any) { + CORE_LOGGER.debug(`[metrics] gpu: collection failed: ${e?.message}`) + return undefined + } + } + + dispose(): void { + for (const c of this.collectors.values()) { + try { + c.dispose() + } catch { + // best-effort + } + } + this.collectors.clear() + } +} diff --git a/src/components/c2d/gpu/nvml.ts b/src/components/c2d/gpu/nvml.ts new file mode 100644 index 000000000..122ddda58 --- /dev/null +++ b/src/components/c2d/gpu/nvml.ts @@ -0,0 +1,201 @@ +import type { ComputeResource } from '../../../@types/C2D/C2D.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { + GpuDeviceHandle, + GpuDeviceMetrics, + GpuVendorCollector, + parseMemoryTotalToBytes +} from './types.js' + +// NVIDIA GPU metrics via NVML (libnvidia-ml.so.1), loaded through the koffi prebuilt-free +// FFI. koffi is an OPTIONAL dependency and the shared library is only present on NVIDIA +// hosts, so every interaction is defensive: a load/init failure disables the backend +// (detect() → false) with a single warn log, and a per-device sample failure yields nulls +// for that device. Nothing here can throw into the C2D state-machine loop. +// +// NVML is loaded lazily via a non-literal specifier so tsc does not require the module at +// build time (koffi may be absent) and a pure-CPU node never touches it. + +const NVML_SUCCESS = 0 +const NVML_TEMPERATURE_GPU = 0 + +interface NvmlBindings { + deviceType: any + utilizationType: any + memoryType: any + init: (...a: any[]) => number + shutdown: (...a: any[]) => number + getHandleByUUID: (...a: any[]) => number + getUtilizationRates: (...a: any[]) => number + getMemoryInfo: (...a: any[]) => number + getTemperature: (...a: any[]) => number + getPowerUsage: (...a: any[]) => number +} + +export class NvmlGpuCollector implements GpuVendorCollector { + public readonly vendor = 'nvidia' as const + private bindings: NvmlBindings | null = null + private detected: boolean | null = null // null = not yet probed + private initialized = false + + private async loadKoffi(): Promise { + try { + // Non-literal specifier: tsc treats this as `any` and does not resolve it at build + // time, so the (optional) koffi dependency is only required at runtime on GPU hosts. + const specifier = 'koffi' + // eslint-disable-next-line @typescript-eslint/no-implied-eval + const mod: any = await import(specifier) + return mod?.default ?? mod + } catch (e: any) { + CORE_LOGGER.warn( + `GPU metrics (nvidia): koffi FFI not available — NVIDIA GPU metrics disabled (${e?.message}). ` + + `Install the 'koffi' optional dependency to enable them.` + ) + return null + } + } + + private buildBindings(koffi: any): NvmlBindings | null { + try { + const lib = koffi.load('libnvidia-ml.so.1') + const deviceType = koffi.pointer(koffi.opaque()) + const utilizationType = koffi.struct('nvmlUtilization_t', { + gpu: 'uint32', + memory: 'uint32' + }) + const memoryType = koffi.struct('nvmlMemory_t', { + total: 'uint64', + free: 'uint64', + used: 'uint64' + }) + return { + deviceType, + utilizationType, + memoryType, + init: lib.func('nvmlInit_v2', 'int', []), + shutdown: lib.func('nvmlShutdown', 'int', []), + getHandleByUUID: lib.func('nvmlDeviceGetHandleByUUID', 'int', [ + 'string', + koffi.out(koffi.pointer(deviceType)) + ]), + getUtilizationRates: lib.func('nvmlDeviceGetUtilizationRates', 'int', [ + deviceType, + koffi.out(koffi.pointer(utilizationType)) + ]), + getMemoryInfo: lib.func('nvmlDeviceGetMemoryInfo', 'int', [ + deviceType, + koffi.out(koffi.pointer(memoryType)) + ]), + getTemperature: lib.func('nvmlDeviceGetTemperature', 'int', [ + deviceType, + 'int', + koffi.out(koffi.pointer('uint32')) + ]), + getPowerUsage: lib.func('nvmlDeviceGetPowerUsage', 'int', [ + deviceType, + koffi.out(koffi.pointer('uint32')) + ]) + } + } catch (e: any) { + CORE_LOGGER.warn( + `GPU metrics (nvidia): could not bind libnvidia-ml.so.1 — NVIDIA GPU metrics disabled (${e?.message})` + ) + return null + } + } + + async detect(): Promise { + if (this.detected !== null) return this.detected + const koffi = await this.loadKoffi() + if (!koffi) return (this.detected = false) + const bindings = this.buildBindings(koffi) + if (!bindings) return (this.detected = false) + try { + const rc = bindings.init() + if (rc !== NVML_SUCCESS) { + CORE_LOGGER.warn(`GPU metrics (nvidia): nvmlInit failed (code ${rc}) — disabled`) + return (this.detected = false) + } + this.initialized = true + this.bindings = bindings + return (this.detected = true) + } catch (e: any) { + CORE_LOGGER.warn(`GPU metrics (nvidia): nvmlInit threw — disabled (${e?.message})`) + return (this.detected = false) + } + } + + // Resolves a GPU ComputeResource into a handle. NVIDIA devices are pinned by NVML UUID in + // init.deviceRequests.DeviceIDs — we take the first UUID-shaped id ("GPU-…"). + resolve(res: ComputeResource): GpuDeviceHandle | null { + const deviceIds: string[] = res?.init?.deviceRequests?.DeviceIDs ?? [] + const uuid = deviceIds.find((id) => /^GPU-/i.test(id)) ?? deviceIds[0] + return { + resourceId: String(res.id), + vendor: 'nvidia', + uuid, + memoryTotalBytes: parseMemoryTotalToBytes(res.memoryTotal), + shareable: res.shareable === true + } + } + + private sampleOne(handle: GpuDeviceHandle): GpuDeviceMetrics { + const base: GpuDeviceMetrics = { + resourceId: handle.resourceId, + vendor: 'nvidia', + utilizationPercent: null, + memoryUsedBytes: null, + memoryTotalBytes: handle.memoryTotalBytes ?? null, + shared: handle.shareable || undefined + } + const b = this.bindings + if (!b || !handle.uuid) return base + try { + const devOut: any[] = [null] + if (b.getHandleByUUID(handle.uuid, devOut) !== NVML_SUCCESS) return base + const device = devOut[0] + + const util: any = {} + if (b.getUtilizationRates(device, util) === NVML_SUCCESS) { + base.utilizationPercent = Number(util.gpu) + } + const mem: any = {} + if (b.getMemoryInfo(device, mem) === NVML_SUCCESS) { + base.memoryUsedBytes = Number(mem.used) + base.memoryTotalBytes = Number(mem.total) + } + const tempOut: any[] = [0] + if (b.getTemperature(device, NVML_TEMPERATURE_GPU, tempOut) === NVML_SUCCESS) { + base.temperatureC = Number(tempOut[0]) + } + const powerOut: any[] = [0] + if (b.getPowerUsage(device, powerOut) === NVML_SUCCESS) { + base.powerWatts = Number((Number(powerOut[0]) / 1000).toFixed(1)) // mW → W + } + } catch (e: any) { + CORE_LOGGER.debug( + `GPU metrics (nvidia): sample of ${handle.resourceId} failed: ${e?.message}` + ) + } + return base + } + + async sample(handles: GpuDeviceHandle[]): Promise { + if (!(await this.detect())) return [] + return handles.map((h) => this.sampleOne(h)) + } + + dispose(): void { + try { + if (this.initialized && this.bindings) this.bindings.shutdown() + } catch { + // best-effort + } + this.initialized = false + // Clear the cached probe result and release the (now shut-down) bindings so a later + // detect() re-initializes from scratch and sample()/sampleOne() can never use a torn-down + // NVML handle. + this.detected = null + this.bindings = null + } +} diff --git a/src/components/c2d/gpu/types.ts b/src/components/c2d/gpu/types.ts new file mode 100644 index 000000000..401b231b0 --- /dev/null +++ b/src/components/c2d/gpu/types.ts @@ -0,0 +1,63 @@ +import type { ComputeResource } from '../../../@types/C2D/C2D.js' + +export type GpuVendor = 'nvidia' | 'amd' | 'intel' + +// A resolved, ready-to-sample physical GPU, derived once from a job's GPU ComputeResource. +export interface GpuDeviceHandle { + resourceId: string // 'gpu0' — the id jobs request + vendor: GpuVendor + uuid?: string // nvidia: NVML UUID from init.deviceRequests.DeviceIDs + drmCard?: string // amd/intel: 'card0' from init.advanced.Devices (future backends) + memoryTotalBytes?: number // parsed from ComputeResource.memoryTotal, as a fallback + shareable: boolean +} + +// One live sample for one device. Nulls (not zeros) mean "backend could not provide it" so +// consumers never mistake an unreadable metric for an idle device. +export interface GpuDeviceMetrics { + resourceId: string + vendor: GpuVendor + utilizationPercent: number | null + memoryUsedBytes: number | null + memoryTotalBytes: number | null + temperatureC?: number + powerWatts?: number + shared?: boolean +} + +// Vendor backend contract. Only the NVIDIA (NVML) backend is implemented today; AMD/Intel +// are interface slots whose detect() returns false until their backends land. +export interface GpuVendorCollector { + readonly vendor: GpuVendor + detect(): Promise // is this backend usable on this host? Run once, cached. + resolve(res: ComputeResource): GpuDeviceHandle | null + sample(handles: GpuDeviceHandle[]): Promise + dispose(): void +} + +// Parses a ComputeResource.memoryTotal string ("3072 MiB", "40960 MiB", "16 GiB", "8GB") +// into bytes. Returns undefined when absent or unparseable — callers treat it as unknown. +export function parseMemoryTotalToBytes(memoryTotal?: string): number | undefined { + if (!memoryTotal) return undefined + const trimmed = String(memoryTotal).trim() + // eslint-disable-next-line security/detect-unsafe-regex + const m = trimmed.match(/^(\d+(?:\.\d+)?)\s*([a-zA-Z]{1,3})?$/) + if (!m) return undefined + const value = parseFloat(m[1]) + if (!Number.isFinite(value)) return undefined + const unit = (m[2] || 'B').toLowerCase() + const factors: Record = { + b: 1, + kb: 1e3, + mb: 1e6, + gb: 1e9, + tb: 1e12, + kib: 1024, + mib: 1024 ** 2, + gib: 1024 ** 3, + tib: 1024 ** 4 + } + const factor = factors[unit] + if (!factor) return undefined + return Math.round(value * factor) +} diff --git a/src/components/c2d/index.ts b/src/components/c2d/index.ts index 70ed3e235..4816def3b 100644 --- a/src/components/c2d/index.ts +++ b/src/components/c2d/index.ts @@ -2,7 +2,8 @@ import { deleteKeysFromObject, sanitizeServiceFiles } from '../../utils/util.js' import { BaseFileObject, EncryptMethod } from '../../@types/fileObject.js' import { CORE_LOGGER } from '../../utils/logging/common.js' -import { ComputeJob, DBComputeJob } from '../../@types/index.js' +import { DBComputeJob } from '../../@types/index.js' +import type { ContainerMetricsSnapshot, PublicComputeJob } from '../../@types/C2D/C2D.js' import { OceanNode } from '../../OceanNode.js' export { C2DEngine } from './compute_engine_base.js' @@ -31,8 +32,28 @@ export async function decryptFilesObject( } } -export function omitDBComputeFieldsFromComputeJob(dbCompute: DBComputeJob): ComputeJob { - const job: ComputeJob = deleteKeysFromObject(dbCompute, [ +// Returns a runtime-metrics snapshot safe to expose in a response: drops the internal `prev` +// accumulator (delta bookkeeping meaningless to callers). Returns undefined for a missing snapshot. +export function sanitizePublicMetrics( + snapshot: ContainerMetricsSnapshot | undefined +): ContainerMetricsSnapshot | undefined { + if (!snapshot) return undefined + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { prev, ...pub } = snapshot + return pub as ContainerMetricsSnapshot +} + +// Maps a DBComputeJob to the public ComputeJob shape by stripping node-internal fields. +// +// By DEFAULT `runtimeMetrics` is stripped — this same default-shape is serialized into the +// on-chain escrow claim proof (compute_engine_docker.ts), which MUST stay deterministic and +// metrics-free. Pass { includeMetrics: true } ONLY on the authenticated owner status path to +// keep a sanitized snapshot; never for the proof. +export function omitDBComputeFieldsFromComputeJob( + dbCompute: DBComputeJob, + opts: { includeMetrics?: boolean } = {} +): PublicComputeJob { + const keysToOmit = [ 'clusterHash', 'configlogURL', 'publishlogURL', @@ -46,6 +67,11 @@ export function omitDBComputeFieldsFromComputeJob(dbCompute: DBComputeJob): Comp 'encryptedDockerRegistryAuth', 'output', 'outputBucketId' - ]) as ComputeJob + ] + if (!opts.includeMetrics) keysToOmit.push('runtimeMetrics') + const job = deleteKeysFromObject(dbCompute, keysToOmit) as PublicComputeJob + if (opts.includeMetrics && dbCompute.runtimeMetrics) { + job.runtimeMetrics = sanitizePublicMetrics(dbCompute.runtimeMetrics) + } return job } diff --git a/src/components/core/admin/adminHandler.ts b/src/components/core/admin/adminHandler.ts index ec2502fa2..ed90138ab 100644 --- a/src/components/core/admin/adminHandler.ts +++ b/src/components/core/admin/adminHandler.ts @@ -13,12 +13,16 @@ import { P2PCommandResponse } from '../../../@types/OceanNode.js' import { ReadableString } from '../../P2P/handleProtocolCommands.js' import { CommonValidation } from '../../../utils/validators.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { normalizeCommandAddresses } from '../../../utils/evmAddress.js' export abstract class AdminCommandHandler extends BaseHandler implements IValidateAdminCommandHandler { async verifyParamsAndRateLimits(task: SignedCommand): Promise { + // Same ingress normalization as CommandHandler: the admin `address` is matched against + // ALLOWED_ADMINS / access lists, so its casing must not decide whether a call is admin. + normalizeCommandAddresses(task) if (!(await this.checkRateLimit(task.caller))) { return buildRateLimitReachedResponse() } diff --git a/src/components/core/compute/getStatus.ts b/src/components/core/compute/getStatus.ts index 946931286..218f502aa 100644 --- a/src/components/core/compute/getStatus.ts +++ b/src/components/core/compute/getStatus.ts @@ -11,6 +11,31 @@ import { } from '../../httpRoutes/validateCommands.js' import { isAddress } from 'ethers' +// How hard the node should try to attach runtime metrics to a status response. +// 'off' — do not attach, do not spend an auth round-trip on it +// 'best-effort' — attach if ownership verifies, stay silent (200, no metrics) if it does not +// 'required' — the caller explicitly asked: answer 400/401 rather than trimming silently +export type MetricsRequestMode = 'off' | 'best-effort' | 'required' + +/** + * Runtime metrics are owner-only, and ON BY DEFAULT: a caller that proves control of + * consumerAddress gets them without having to ask for them. `includeMetrics` only overrides + * that default. + * + * The default is deliberately 'best-effort' rather than 'required': COMPUTE_GET_STATUS is + * also a legitimate UNAUTHENTICATED call, so demanding credentials by default would break + * every client that just polls a jobId. + */ +export function resolveMetricsRequestMode( + task: ComputeGetStatusCommand +): MetricsRequestMode { + if (task.includeMetrics === false) return 'off' + if (task.includeMetrics === true) return 'required' + const hasOwnerCredentials = + !!task.consumerAddress && (!!task.authorization || (!!task.signature && !!task.nonce)) + return hasOwnerCredentials ? 'best-effort' : 'off' +} + export class ComputeGetStatusHandler extends CommandHandler { validate(command: ComputeGetStatusCommand): ValidateParams { const validation = validateCommandParameters(command, []) @@ -34,6 +59,35 @@ export class ComputeGetStatusHandler extends CommandHandler { return validationResponse } try { + const metricsMode = resolveMetricsRequestMode(task) + let includeMetrics = false + if (metricsMode !== 'off') { + if (!task.consumerAddress) { + // only reachable in 'required' mode — 'best-effort' needs credentials to be set + return { + stream: null, + status: { + httpStatus: 400, + error: 'includeMetrics requires consumerAddress' + } + } + } + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus === 200) { + includeMetrics = true + } else if (metricsMode === 'required') { + // Fail the whole request only when metrics were explicitly asked for; a failed + // opportunistic check just means no metrics. + return auth + } + } + const response: ComputeJob[] = [] // two scenarios here: // 1. if we have a jobId, then we know what C2D Cluster to query @@ -59,7 +113,8 @@ export class ComputeGetStatusHandler extends CommandHandler { const jobs = await engine.getComputeJobStatus( task.consumerAddress, task.agreementId, - jobId + jobId, + includeMetrics ) if (jobs && jobs.length > 0) response.push(...jobs) diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 3c0b1daeb..b664df533 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -14,6 +14,7 @@ import { import { CORE_LOGGER } from '../../../utils/logging/common.js' import { ReadableString } from '../../P2P/handlers.js' import { CONNECTION_HISTORY_DELETE_THRESHOLD } from '../../../utils/constants.js' +import { normalizeCommandAddresses, sameAddress } from '../../../utils/evmAddress.js' export abstract class BaseHandler implements ICommandHandler { public nodeInstance: OceanNode @@ -157,6 +158,10 @@ export abstract class CommandHandler { abstract validate(command: Command): ValidateParams async verifyParamsAndRateLimits(task: Command): Promise { + // Canonicalize every caller-supplied address (EIP-55) before ANYTHING reads it: the + // validators below, DB lookups keyed by owner, ownership comparisons. Addresses are + // identity keys, and a differently-cased one silently misses instead of erroring. + normalizeCommandAddresses(task) // first check rate limits, if any if (!(await this.checkRateLimit(task.caller))) { return buildRateLimitReachedResponse() @@ -202,6 +207,19 @@ export abstract class CommandHandler status: { httpStatus: 401, error: isAuthRequestValid.error } } } + // An auth TOKEN authenticates whoever it was issued to — not necessarily the address the + // command claims. Bind the two: without this, a valid token for address A would authorize a + // request naming address B as its consumerAddress, i.e. read/act on B's jobs, services and + // buckets. (The signature path returns the very address it verified, so it always matches.) + if (address && !sameAddress(isAuthRequestValid.address, address)) { + return { + stream: null, + status: { + httpStatus: 401, + error: 'Authenticated address does not match the requested address' + } + } + } return { stream: null, diff --git a/src/components/core/service/getStatus.ts b/src/components/core/service/getStatus.ts index a141b3e40..72def8e57 100644 --- a/src/components/core/service/getStatus.ts +++ b/src/components/core/service/getStatus.ts @@ -45,8 +45,16 @@ export class ServiceGetStatusHandler extends CommandHandler { jobs.push(...(await eng.getServiceStatus(task.consumerAddress, task.serviceId))) } + // Ownership is already proven above (this command is always authenticated), so runtime + // metrics are included BY DEFAULT here — only an explicit includeMetrics=false opts out. return { - stream: Readable.from(JSON.stringify(jobs.map(toPublicServiceJob))), + stream: Readable.from( + JSON.stringify( + jobs.map((job) => + toPublicServiceJob(job, { includeMetrics: task.includeMetrics !== false }) + ) + ) + ), status: { httpStatus: 200 } } } diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts index d47221ae6..f3b9328f5 100644 --- a/src/components/core/service/utils.ts +++ b/src/components/core/service/utils.ts @@ -5,6 +5,7 @@ import type { KeyManager } from '../../KeyManager/index.js' import type { C2DDatabase } from '../../database/C2DDatabase.js' import type { C2DEngine } from '../../c2d/compute_engine_base.js' import type { C2DEngines } from '../../c2d/compute_engines.js' +import { sanitizePublicMetrics } from '../../c2d/index.js' // Looks up a service job and resolves the engine that OWNS it (by clusterHash). Every // engine shares the same C2DDatabase, so any engine's db returns the job — taking the @@ -54,11 +55,19 @@ export async function decryptUserData( // can pass engine results straight through. EVERY handler returning service jobs // (SERVICE_START / STOP / EXTEND / RESTART / GET_STATUS) must map results through this. export function toPublicServiceJob( - job: ServiceJob | null + job: ServiceJob | null, + opts: { includeMetrics?: boolean } = {} ): Omit | null { if (!job) return null // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { userData, ...pub } = job + const { userData, runtimeMetrics, ...rest } = job + // userData is ALWAYS stripped. The owner-scoped status path may opt in to the sanitized + // runtime metrics (internal `prev` accumulator dropped); otherwise they stay absent. + // SERVICE_LIST uses toListedServiceJob, which never includes metrics. + const pub: Omit = { ...rest } + if (opts.includeMetrics && runtimeMetrics) { + pub.runtimeMetrics = sanitizePublicMetrics(runtimeMetrics) + } return pub } @@ -71,12 +80,18 @@ export function toListedServiceJob( job: ServiceJob | null ): Omit< ServiceJob, - 'userData' | 'dockerCmd' | 'dockerEntrypoint' | 'dockerfile' | 'additionalDockerFiles' + | 'userData' + | 'runtimeMetrics' + | 'dockerCmd' + | 'dockerEntrypoint' + | 'dockerfile' + | 'additionalDockerFiles' > | null { if (!job) return null // eslint-disable-next-line @typescript-eslint/no-unused-vars const { userData, + runtimeMetrics, dockerCmd, dockerEntrypoint, dockerfile, diff --git a/src/components/core/utils/nonceHandler.ts b/src/components/core/utils/nonceHandler.ts index 9e1cd41f8..88f1dad08 100644 --- a/src/components/core/utils/nonceHandler.ts +++ b/src/components/core/utils/nonceHandler.ts @@ -10,6 +10,7 @@ import { PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { NonceCommand } from '../../../@types/commands.js' import { streamToString } from '../../../utils/util.js' import { Readable } from 'node:stream' +import { addressCasingVariants } from '../../../utils/evmAddress.js' export function getDefaultErrorResponse(errorMessage: string): P2PCommandResponse { return { @@ -226,6 +227,38 @@ export async function verifyConsumerSignature( command: string = null, config?: OceanNodeConfig, chainId?: string | null +): Promise { + // The signed message embeds the consumer address as a STRING, so its casing is part of what + // was signed. Addresses are checksum-normalized on ingress (utils/evmAddress.ts), which would + // otherwise reject clients that built and signed the message from the lowercase form — so + // try every plausible casing. Deduped, canonical form first: one attempt for most callers. + for (const candidate of addressCasingVariants(consumer)) { + // eslint-disable-next-line no-await-in-loop + if ( + await verifySignatureForConsumer( + candidate, + nonce, + signature, + issuerPeerId, + command, + config, + chainId + ) + ) { + return true + } + } + return false +} + +async function verifySignatureForConsumer( + consumer: string, + nonce: string | number, + signature: string, + issuerPeerId: string, + command: string = null, + config?: OceanNodeConfig, + chainId?: string | null ): Promise { const message = String( String(consumer) + String(nonce) + String(command) + String(issuerPeerId) diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index b19d30dea..67317b2b1 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -3,7 +3,8 @@ import fs from 'fs' import { ComputeEnvironment, DBComputeJob, - C2DStatusNumber + C2DStatusNumber, + ContainerMetricsSnapshot } from '../../@types/C2D/C2D.js' import { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' import { SQLiteCompute } from './sqliteCompute.js' @@ -88,6 +89,25 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.updateServiceJob(job) } + // Guarded, metrics-only write: persists runtimeMetrics without touching lifecycle fields, and + // only when the row still matches the expected owner/clusterHash/status/containerId. Best-effort. + async updateServiceJobMetrics( + serviceId: string, + expected: { + owner: string + clusterHash: string + status: number + containerId: string + }, + runtimeMetrics: ContainerMetricsSnapshot + ): Promise { + return await this.provider.updateServiceJobMetrics( + serviceId, + expected, + runtimeMetrics + ) + } + async getRunningServiceJobs(clusterHash?: string): Promise { return await this.provider.getRunningServiceJobs(clusterHash) } diff --git a/src/components/database/SQLLiteNonceDatabase.ts b/src/components/database/SQLLiteNonceDatabase.ts index 039518050..0af5d046d 100644 --- a/src/components/database/SQLLiteNonceDatabase.ts +++ b/src/components/database/SQLLiteNonceDatabase.ts @@ -1,6 +1,7 @@ import { OceanNodeDBConfig } from '../../@types/OceanNode.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' +import { addressCasingVariants, normalizeAddress } from '../../utils/evmAddress.js' import { AbstractNonceDatabase } from './BaseDatabase.js' import { SQLiteProvider } from './sqlite.js' import { TypesenseSchema } from './TypesenseSchemas.js' @@ -23,7 +24,7 @@ export class SQLLiteNonceDatabase extends AbstractNonceDatabase { async create(address: string, nonce: number) { try { - return await this.provider.createNonce(address, nonce) + return await this.provider.createNonce(normalizeAddress(address), nonce) } catch (error) { const errorMsg = `Error when creating new nonce entry ${nonce} for address ${address}: ` + @@ -38,9 +39,22 @@ export class SQLLiteNonceDatabase extends AbstractNonceDatabase { } } + // Rows are keyed by the checksummed address. Pre-normalization rows may be keyed by another + // casing of the SAME address, so read every variant and keep the HIGHEST nonce: the row id + // may migrate, but the nonce must stay monotonic — dropping back to 0 would briefly re-open + // the replay window that the nonce exists to close. async retrieve(address: string) { + const id = normalizeAddress(address) try { - return await this.provider.retrieveNonce(address) + let highest: number | null = null + for (const variant of addressCasingVariants(address)) { + // eslint-disable-next-line no-await-in-loop + const row = await this.provider.retrieveNonce(variant) + if (row && row.nonce !== null && (highest === null || row.nonce > highest)) { + highest = row.nonce + } + } + return { id, nonce: highest } } catch (error) { const errorMsg = `Error when retrieving nonce entry for address ${address}: ` + error.message diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 7511350b5..cef4c2f1f 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -2,7 +2,8 @@ import { typesenseSchemas, TypesenseSchema } from './TypesenseSchemas.js' import { C2DStatusNumber, C2DStatusText, - type DBComputeJob + type DBComputeJob, + type ContainerMetricsSnapshot } from '../../@types/C2D/C2D.js' import { ServiceStatusNumber, @@ -56,7 +57,14 @@ function getInternalStructure(job: DBComputeJob): any { outputBucketId: job.outputBucketId, jobIdHash: job.jobIdHash, buildStartTimestamp: job.buildStartTimestamp, - buildStopTimestamp: job.buildStopTimestamp + buildStopTimestamp: job.buildStopTimestamp, + // Runtime metrics snapshot. It MUST round-trip through the body blob: it is what the owner + // reads back on COMPUTE_GET_STATUS, and it is also the previous-sample accumulator every + // next sample needs — without it persisted, every sample is a "first sample", so CPU % can + // never be computed (no delta) and the memory peak never accumulates. Still node-internal + // on the way out: omitDBComputeFieldsFromComputeJob strips it unless the verified owner + // asked for it, so it stays out of the escrow claim proof. + runtimeMetrics: job.runtimeMetrics } return internalBlob } @@ -311,6 +319,74 @@ export class SQLiteCompute implements ComputeDatabaseProvider { } } + // Persists ONLY runtimeMetrics onto a service job — best-effort telemetry that must NOT clobber + // lifecycle fields another process may have changed. The read + validate + merge + write run + // inside a single `BEGIN IMMEDIATE` transaction so the sequence is atomic even across processes + // sharing the SQLite file: IMMEDIATE takes the write lock up front (waiting up to busy_timeout), + // so no other process can commit a lifecycle change (status, expiry, container) between our read + // and our write. Inside the transaction it re-reads the CURRENT row, re-validates + // owner/clusterHash/status/containerId, merges ONLY runtimeMetrics into that current body, and + // writes back ONLY the `body` column guarded on the unchanged status — leaving every lifecycle + // field (incl. expiresAt, in both column and body) exactly as last committed. Returns true when a + // row was written; any mismatch/error rolls back and returns false. + // eslint-disable-next-line require-await + async updateServiceJobMetrics( + serviceId: string, + expected: { + owner: string + clusterHash: string + status: number + containerId: string + }, + runtimeMetrics: ContainerMetricsSnapshot + ): Promise { + try { + this.db.exec('BEGIN IMMEDIATE;') + } catch (err) { + DATABASE_LOGGER.error(`metrics update: could not begin transaction: ${err.message}`) + return false + } + try { + const row = this.db.get<{ body: Uint8Array }>( + `SELECT body FROM service_jobs WHERE serviceId = ?;`, + [serviceId] + ) + if (!row) { + this.db.exec('ROLLBACK;') + return false + } + const body = JSON.parse(Buffer.from(row.body).toString()) as ServiceJob + if ( + body.owner !== expected.owner || + body.clusterHash !== expected.clusterHash || + body.status !== expected.status || + body.containerId !== expected.containerId + ) { + this.db.exec('ROLLBACK;') + return false + } + body.runtimeMetrics = runtimeMetrics + const { changes } = this.db.run( + `UPDATE service_jobs SET body = ? WHERE serviceId = ? AND status = ?;`, + [ + Buffer.from(JSON.stringify({ ...body, updatedAt: Date.now() })), + serviceId, + expected.status + ] + ) + this.db.exec('COMMIT;') + return changes > 0 + } catch (err) { + DATABASE_LOGGER.error(`Error while updating service job metrics: ${err.message}`) + try { + this.db.exec('ROLLBACK;') + } catch { + // best-effort: transaction may already be closed + } + return false + } + } + private mapServiceRows(rows: any[] | undefined): ServiceJob[] { if (!rows || rows.length === 0) return [] // BLOB comes back as Uint8Array from node:sqlite; decode through Buffer before parsing. @@ -326,7 +402,10 @@ export class SQLiteCompute implements ComputeDatabaseProvider { params.push(serviceId) } if (owner) { - selectSQL += ` AND owner = ?` + // COLLATE NOCASE: callers are checksum-normalized on ingress (utils/evmAddress.ts), but + // rows written before that may hold another casing of the same address — and a missed + // match here reads as "no such service" rather than an error. + selectSQL += ` AND owner = ? COLLATE NOCASE` params.push(owner) } try { @@ -562,7 +641,9 @@ export class SQLiteCompute implements ComputeDatabaseProvider { params.push(agreementId) } if (owner) { - selectSQL += ` AND owner = ?` + // COLLATE NOCASE — see getServiceJob: an owner casing mismatch must not read as + // "job not found". + selectSQL += ` AND owner = ? COLLATE NOCASE` params.push(owner) } @@ -751,7 +832,9 @@ export class SQLiteCompute implements ComputeDatabaseProvider { if (consumerAddrs && consumerAddrs.length > 0) { const placeholders = consumerAddrs.map(() => '?').join(',') - conditions.push(`owner IN (${placeholders})`) + // COLLATE NOCASE for the same reason as getJob/getServiceJob: match an owner whatever + // casing the row was written with. + conditions.push(`owner COLLATE NOCASE IN (${placeholders})`) params.push(...consumerAddrs) } diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 3ca98d085..ace836525 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -222,6 +222,15 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/compute`, async (req, res) => { consumerAddress: (req.query.consumerAddress as string) || null, jobId: (req.query.jobId as string) || null, agreementId: (req.query.agreementId as string) || null, + // Owner-only runtime metrics, on by default for authenticated owners. Absent means + // "best-effort" (undefined, not false) — see ComputeGetStatusHandler for the tri-state. + includeMetrics: + req.query.includeMetrics === undefined + ? undefined + : req.query.includeMetrics === 'true', + nonce: (req.query.nonce as string) || null, + signature: (req.query.signature as string) || null, + authorization: req.headers?.authorization || null, caller: req.caller } const response = await new ComputeGetStatusHandler(req.oceanNode).handle( @@ -475,6 +484,12 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceStatus`, async (req, res) => nonce: req.query.nonce as string, signature: req.query.signature as string, serviceId: (req.query.serviceId as string) || undefined, + // On by default (this command is always authenticated + owner-scoped); pass + // includeMetrics=false to opt out. + includeMetrics: + req.query.includeMetrics === undefined + ? undefined + : req.query.includeMetrics === 'true', node: (req.query.node as string) || null, authorization: req.headers?.authorization, caller: req.caller diff --git a/src/test/integration/compute.test.ts b/src/test/integration/compute.test.ts index 96f408a58..cf789ca60 100644 --- a/src/test/integration/compute.test.ts +++ b/src/test/integration/compute.test.ts @@ -1325,6 +1325,64 @@ describe('********** Compute', () => { const jobs = await streamToObject(response.stream as Readable) console.log('Checking FREE job status...') console.log(jobs[0]) + // Backward-compat guard: an UNAUTHENTICATED status call (no consumerAddress credentials) + // never carries runtime metrics, even though metrics are on by default for owners — they + // stay node-internal until ownership is proven. + assert( + !('runtimeMetrics' in jobs[0]), + 'unauthenticated COMPUTE_GET_STATUS must not expose runtimeMetrics' + ) + + // Explicitly requesting metrics is strict: without an authenticated consumerAddress it is + // rejected (never silently downgraded), so the caller learns WHY they are missing. + const gated = await new ComputeGetStatusHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.COMPUTE_GET_STATUS, + consumerAddress: null, + agreementId: null, + jobId: freeJobId, + includeMetrics: true + } as ComputeGetStatusCommand) + assert( + gated.status.httpStatus === 400, + 'includeMetrics without consumerAddress must be rejected' + ) + }) + + it('should get job status (with metrics, by default) for the authenticated owner', async () => { + // Deliberately LOWERCASED: addresses are canonicalized on ingress, so a non-checksummed + // address must resolve to the same job — and the signature, built over the lowercase form + // the client actually used, must still verify. + const lowercasedOwner = (await consumerAccount.getAddress()).toLowerCase() + const nonce = Date.now().toString() + const messageHashBytes = createHashForSignature( + lowercasedOwner, + nonce, + PROTOCOL_COMMANDS.COMPUTE_GET_STATUS + ) + const signature = await safeSign(consumerAccount, messageHashBytes) + const response = await new ComputeGetStatusHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.COMPUTE_GET_STATUS, + consumerAddress: lowercasedOwner, + agreementId: null, + jobId: freeJobId, + nonce, + signature + } as ComputeGetStatusCommand) + assert( + response.status.httpStatus === 200, + `expected 200, got ${response.status.httpStatus}: ${response.status?.error ?? ''}` + ) + const ownedJobs = await streamToObject(response.stream as Readable) + assert( + ownedJobs.length === 1, + 'a lowercased consumerAddress must still match the owner of the job' + ) + // Metrics need no flag for the verified owner. The snapshot appears once the container has + // been sampled at least once (C2D_METRICS_INTERVAL_SECONDS), and never carries the + // node-internal delta accumulator. + if (ownedJobs[0].runtimeMetrics) { + expect('prev' in ownedJobs[0].runtimeMetrics).to.equal(false) + } }) // algo and checksums related describe('C2D algo and checksums related', () => { diff --git a/src/test/integration/database.test.ts b/src/test/integration/database.test.ts index 519d378e0..2de93c3bb 100644 --- a/src/test/integration/database.test.ts +++ b/src/test/integration/database.test.ts @@ -157,6 +157,19 @@ describe('********** NonceDatabase CRUD (without Elastic or Typesense co expect(result?.id).to.equal('0x456') expect(result?.nonce).to.equal(1) }) + + it('keys rows by the checksummed address and reads them back in any casing', async () => { + // One address = one nonce row. If casing forked the key, a client alternating between + // the lowercase and checksummed forms would carry two independent nonce counters. + const checksummed = '0x7C8226E267Cd509bCBE12B4e69fbE07052422Dbd' + const created = await database.nonce.create(checksummed.toLowerCase(), 7) + expect(created?.id).to.equal(checksummed) + expect((await database.nonce.retrieve(checksummed))?.nonce).to.equal(7) + expect((await database.nonce.retrieve(checksummed.toLowerCase()))?.nonce).to.equal(7) + const upper = '0x' + checksummed.slice(2).toUpperCase() + expect((await database.nonce.retrieve(upper))?.nonce).to.equal(7) + await database.nonce.delete(checksummed) + }) }) describe('********** IndexerDatabase CRUD', () => { diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index 585e477e5..bbca9c93f 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -478,6 +478,12 @@ describe('********** Service on Demand', () => { expect(job.serviceId).to.equal(serviceId) expect((job as any).userData).to.equal(undefined) assert(job.payment, 'payment should be present') + // This command is always authenticated + owner-scoped, so runtime metrics come back BY + // DEFAULT — and when a snapshot has been sampled it is exposed WITHOUT the internal + // `prev` accumulator. + if ((job as any).runtimeMetrics) { + expect('prev' in (job as any).runtimeMetrics).to.equal(false) + } // an unauthenticated status request (no nonce/signature) is rejected const unauth = await new ServiceGetStatusHandler(oceanNode).handle({ @@ -486,6 +492,25 @@ describe('********** Service on Demand', () => { serviceId } as ServiceGetStatusCommand) expect(unauth.status.httpStatus).to.not.equal(200) + + // opt-OUT: includeMetrics=false keeps metrics off the response entirely + const { nonce, signature } = await signFor( + consumerAccount, + PROTOCOL_COMMANDS.SERVICE_GET_STATUS + ) + const noMetrics = await new ServiceGetStatusHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + serviceId, + consumerAddress, + nonce, + signature, + includeMetrics: false + } as ServiceGetStatusCommand) + expect(noMetrics.status.httpStatus).to.equal(200) + const [noMetricsJob] = (await streamToObject( + noMetrics.stream as Readable + )) as ServiceJob[] + expect((noMetricsJob as any).runtimeMetrics).to.equal(undefined) }) it('(e2) SERVICE_LIST returns the node-wide resource-holding set (not owner-scoped)', async () => { @@ -513,6 +538,7 @@ describe('********** Service on Demand', () => { expect((listed as any).userData).to.equal(undefined) expect((listed as any).dockerCmd).to.equal(undefined) expect((listed as any).dockerEntrypoint).to.equal(undefined) + expect((listed as any).runtimeMetrics).to.equal(undefined) // status filter: Running includes the service, Expired does not const sig2 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 2cc4f6e5b..23f2fb02a 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -226,6 +226,49 @@ describe('Compute Jobs Database', () => { expect(updatedJob.statusText).to.be.equal(C2DStatusText.PullImage) }) + it('round-trips runtimeMetrics through updateJob/getJob', async () => { + // The snapshot MUST survive the body blob. It is what the owner reads back on + // COMPUTE_GET_STATUS, and it is the previous-sample accumulator the next sample needs to + // compute CPU % — if it is dropped on write, every sample is a "first sample" and CPU is + // permanently 0. + const [job] = await db.getJob(jobId) + job.runtimeMetrics = { + collectedAt: '2026-08-13T18:31:03.083Z', + containerState: { status: 'running', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 97.5, + allocated: 3, + usagePercentOfAllocated: 32.5, + cumulativeSeconds: 42.25, + throttledPeriods: 2, + throttledSeconds: 0.5 + }, + memory: { + usageBytes: 1024, + limitBytes: 4096, + usagePercent: 25, + peakUsageBytes: 2048 + }, + disk: { usedBytes: 512, quotaBytes: 1024, usagePercent: 50 }, + blockIO: { readBytes: 10, writeBytes: 20 }, + pids: { current: 3, limit: 512 }, + prev: { + cpuTotal: 42_250_000_000, + systemCpu: 100e9, + sampledAt: '2026-08-13T18:31:03.083Z' + } + } + expect(await db.updateJob(job)).to.be.equal(1) + + const [reloaded] = await db.getJob(jobId) + assert(reloaded.runtimeMetrics, 'runtimeMetrics must be persisted on the job record') + expect(reloaded.runtimeMetrics.cpu.cumulativeSeconds).to.be.equal(42.25) + expect(reloaded.runtimeMetrics.memory.peakUsageBytes).to.be.equal(2048) + expect(reloaded.runtimeMetrics.disk.usedBytes).to.be.equal(512) + // the delta accumulator has to survive too, otherwise CPU % can never be computed + expect(reloaded.runtimeMetrics.prev?.cpuTotal).to.be.equal(42_250_000_000) + }) + it('should get running jobs', async () => { const job: DBComputeJob = { owner: '0xe2DD09d719Da89e5a3D0F2549c7E24566e947261', @@ -2792,3 +2835,155 @@ describe('service start/restart Docker cleanup on failure', function () { ).to.equal(true) }) }) + +// The admin-facing view of the metrics: one roll-up line per sampling interval, plus a +// "pressure" line per workload close to a limit. Worth testing because it is pure formatting +// inside a try/catch — a throw here would be swallowed and the admin would just see nothing. +describe('engine-wide metrics summary logging', function () { + let engine: any + let debugSpy: sinon.SinonSpy + + function snapshot(overrides: any = {}): any { + return { + collectedAt: new Date().toISOString(), + containerState: { status: 'running', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 50, + allocated: 2, + usagePercentOfAllocated: 25, + cumulativeSeconds: 10, + throttledPeriods: 0, + throttledSeconds: 0 + }, + memory: { + usageBytes: 100 * 1024 * 1024, + limitBytes: 1024 * 1024 * 1024, + usagePercent: 9.77, + peakUsageBytes: 120 * 1024 * 1024 + }, + disk: { usedBytes: 50 * 1024 * 1024 }, + blockIO: { readBytes: 0, writeBytes: 0 }, + pids: { current: 5, limit: 512 }, + ...overrides + } + } + + beforeEach(function () { + // Same pattern as the other engine unit tests: prototype without the Docker constructor. + engine = Object.create(C2DEngineDocker.prototype) + engine.getC2DConfig = sinon.stub().returns({ hash: 'cluster-hash' }) + engine.physicalLimits = new Map([['cpu', 8]]) + engine.lastMetricsSummaryAt = 0 + debugSpy = sinon.spy(CORE_LOGGER, 'debug') + }) + + afterEach(function () { + debugSpy.restore() + }) + + const summaryLines = () => + debugSpy + .getCalls() + .map((c) => String(c.args[0])) + .filter((m) => m.includes('[metrics] summary')) + + const pressureLines = () => + debugSpy + .getCalls() + .map((c) => String(c.args[0])) + .filter((m) => m.includes('[metrics] pressure')) + + it('rolls every sampled job and service into one line', function () { + engine.logMetricsSummary( + [ + { jobId: 'job-1', runtimeMetrics: snapshot() }, + { jobId: 'job-2', runtimeMetrics: snapshot() } + ], + [{ serviceId: 'svc-1', runtimeMetrics: snapshot() }] + ) + const [line] = summaryLines() + expect(line, 'a summary line must be logged').to.not.equal(undefined) + expect(line).to.contain('engine cluster-hash') + expect(line).to.contain('2 job(s) / 1 service(s), 3 sampled') + expect(line).to.contain('cpu 150.0% of host (8 core(s))') // 3 x 50% + expect(line).to.contain('6 core(s) allocated') // 3 x 2 + expect(line).to.contain('mem 300.0 MiB/3.0 GiB allocated') // 3 x 100MiB / 3 x 1GiB + expect(line).to.contain('disk 150.0 MiB') + expect(pressureLines()).to.deep.equal([]) + }) + + it('reports running-but-not-yet-sampled instead of staying silent', function () { + engine.logMetricsSummary([{ jobId: 'job-1' }], []) + const [line] = summaryLines() + expect(line).to.contain('1 job(s) / 0 service(s) running, none sampled yet') + }) + + it('logs nothing at all when nothing is running', function () { + engine.logMetricsSummary([], []) + expect(summaryLines()).to.deep.equal([]) + }) + + it('flags workloads near their limits', function () { + engine.logMetricsSummary( + [ + { + jobId: 'job-hot', + runtimeMetrics: snapshot({ + memory: { + usageBytes: 990 * 1024 * 1024, + limitBytes: 1024 * 1024 * 1024, + usagePercent: 96.7, + peakUsageBytes: 1000 * 1024 * 1024 + }, + disk: { + usedBytes: 95 * 1024 * 1024, + quotaBytes: 100 * 1024 * 1024, + usagePercent: 95 + }, + pids: { current: 500, limit: 512 }, + cpu: { + usagePercent: 199, + allocated: 2, + usagePercentOfAllocated: 99.5, + cumulativeSeconds: 30, + throttledPeriods: 42, + throttledSeconds: 1.5 + } + }) + } + ], + [] + ) + const [line] = pressureLines() + expect(line, 'a pressure line must be logged').to.not.equal(undefined) + expect(line).to.contain('pressure job job-hot') + expect(line).to.contain('mem 96.7% of limit') + expect(line).to.contain('disk 95% of quota') + expect(line).to.contain('pids 500/512') + expect(line).to.contain('cpu throttled 42 periods / 1.5s') + }) + + it('is throttled to the sampling interval (the loop ticks far more often)', function () { + const jobs = [{ jobId: 'job-1', runtimeMetrics: snapshot() }] + engine.logMetricsSummary(jobs, []) + engine.logMetricsSummary(jobs, []) + engine.logMetricsSummary(jobs, []) + expect(summaryLines().length).to.equal(1) + }) + + it('survives a stubbed-out service list (undefined) without throwing', function () { + engine.logMetricsSummary([{ jobId: 'job-1', runtimeMetrics: snapshot() }], undefined) + expect(summaryLines().length).to.equal(1) + }) + + it('logs nothing when collection is disabled', function () { + const original = ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = '0' + try { + engine.logMetricsSummary([{ jobId: 'job-1', runtimeMetrics: snapshot() }], []) + expect(summaryLines()).to.deep.equal([]) + } finally { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = original + } + }) +}) diff --git a/src/test/unit/containerMetrics.test.ts b/src/test/unit/containerMetrics.test.ts new file mode 100644 index 000000000..b449de28c --- /dev/null +++ b/src/test/unit/containerMetrics.test.ts @@ -0,0 +1,679 @@ +import { expect } from 'chai' +import { + buildSnapshot, + describeSnapshot, + formatBytes, + getMetricsIntervalSeconds, + isMetricsCollectionEnabled, + isSnapshotStale, + RawContainerSample +} from '../../components/c2d/containerMetrics.js' +import { parseMemoryTotalToBytes } from '../../components/c2d/gpu/types.js' +import { NvmlGpuCollector } from '../../components/c2d/gpu/nvml.js' +import { + omitDBComputeFieldsFromComputeJob, + sanitizePublicMetrics +} from '../../components/c2d/index.js' +import { + toPublicServiceJob, + toListedServiceJob +} from '../../components/core/service/utils.js' +import type { + ContainerMetricsSnapshot, + DBComputeJob, + ComputeResource +} from '../../@types/C2D/C2D.js' +import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' +import { ENVIRONMENT_VARIABLES, PROTOCOL_COMMANDS } from '../../utils/constants.js' +import { resolveMetricsRequestMode } from '../../components/core/compute/getStatus.js' + +const MB = 1024 * 1024 +const GB = 1024 * MB + +// A cgroup-v2-shaped Docker stats blob (one-shot: no precpu_stats). +function cgroupV2Stats(overrides: any = {}): any { + return { + cpu_stats: { + cpu_usage: { total_usage: 2e9 }, + system_cpu_usage: 20e9, + online_cpus: 4, + throttling_data: { throttled_periods: 5, throttled_time: 2e9 } + }, + memory_stats: { + usage: 500 * MB, + limit: 1024 * MB, + stats: { inactive_file: 100 * MB } + }, + blkio_stats: { + io_service_bytes_recursive: [ + { op: 'Read', value: 1000 }, + { op: 'Write', value: 2000 } + ] + }, + networks: { eth0: { rx_bytes: 111, tx_bytes: 222 } }, + pids_stats: { current: 7 }, + ...overrides + } +} + +function runningState(overrides: any = {}): any { + return { + Status: 'running', + Running: true, + StartedAt: '2026-07-28T10:00:00Z', + OOMKilled: false, + ExitCode: 0, + RestartCount: 0, + ...overrides + } +} + +const ALLOC = { cpu: 2, ramBytes: 1024 * MB, diskBytes: 10 * GB } + +describe('containerMetrics.buildSnapshot', () => { + it('computes CPU % from the previous accumulator (one-shot deltas)', () => { + const prev: Partial = { + collectedAt: '2026-07-28T10:00:00Z', + prev: { cpuTotal: 1e9, systemCpu: 10e9, sampledAt: '2026-07-28T10:00:00Z' } + } + const raw: RawContainerSample = { stats: cgroupV2Stats(), state: runningState() } + const snap = buildSnapshot(raw, prev as ContainerMetricsSnapshot, ALLOC, 5 * GB) + // Δcpu=1e9, Δsys=10e9, cpus=4 → 0.1*4*100 = 40% + expect(snap.cpu.usagePercent).to.equal(40) + expect(snap.cpu.usagePercentOfAllocated).to.equal(20) // 40 / 2 cores + expect(snap.cpu.cumulativeSeconds).to.equal(2) // 2e9 ns + expect(snap.cpu.throttledPeriods).to.equal(5) + expect(snap.cpu.throttledSeconds).to.equal(2) + }) + + it('reports 0% CPU on a true first sample (no prev, no precpu)', () => { + const raw: RawContainerSample = { stats: cgroupV2Stats(), state: runningState() } + const snap = buildSnapshot(raw, undefined, ALLOC, 5 * GB) + expect(snap.cpu.usagePercent).to.equal(0) + // but it still records the accumulator for the next sample + expect(snap.prev?.cpuTotal).to.equal(2e9) + expect(snap.prev?.systemCpu).to.equal(20e9) + }) + + it('subtracts inactive_file for memory (cgroup v2 convention) and uses allocated limit', () => { + const raw: RawContainerSample = { stats: cgroupV2Stats(), state: runningState() } + const snap = buildSnapshot(raw, undefined, ALLOC, 5 * GB) + expect(snap.memory.usageBytes).to.equal(400 * MB) // 500 - 100 inactive + expect(snap.memory.limitBytes).to.equal(1024 * MB) // from allocation + expect(snap.memory.usagePercent).to.be.closeTo(39.06, 0.1) + }) + + it('tracks the memory peak across samples (cgroup v2 has no max_usage)', () => { + const prev = { + collectedAt: '2026-07-28T10:00:00Z', + memory: { usageBytes: 0, limitBytes: 0, usagePercent: 0, peakUsageBytes: 600 * MB } + } as ContainerMetricsSnapshot + const raw: RawContainerSample = { stats: cgroupV2Stats(), state: runningState() } + const snap = buildSnapshot(raw, prev, ALLOC, 5 * GB) + // current usage 400MB < previous peak 600MB → peak retained + expect(snap.memory.peakUsageBytes).to.equal(600 * MB) + }) + + it('reports disk usage vs quota when a disk allocation is present', () => { + const raw: RawContainerSample = { stats: cgroupV2Stats(), state: runningState() } + const snap = buildSnapshot(raw, undefined, ALLOC, 5 * GB) + expect(snap.disk.usedBytes).to.equal(5 * GB) + expect(snap.disk.quotaBytes).to.equal(10 * GB) + expect(snap.disk.usagePercent).to.equal(50) + }) + + it('falls back to SizeRw for disk when no du() figure is supplied (services)', () => { + const raw: RawContainerSample = { + stats: cgroupV2Stats(), + state: runningState({ SizeRw: 3 * MB }) + } + const snap = buildSnapshot( + raw, + undefined, + { cpu: 1, ramBytes: 0, diskBytes: 0 }, + undefined + ) + expect(snap.disk.usedBytes).to.equal(3 * MB) + expect(snap.disk.quotaBytes).to.equal(undefined) + }) + + it('omits the network field when the container has no networks (NetworkMode none)', () => { + const raw: RawContainerSample = { + stats: cgroupV2Stats({ networks: undefined }), + state: runningState() + } + const snap = buildSnapshot(raw, undefined, ALLOC, 0) + expect(snap.network).to.equal(undefined) + }) + + it('sums block IO and network across interfaces', () => { + const raw: RawContainerSample = { + stats: cgroupV2Stats({ + networks: { + eth0: { rx_bytes: 100, tx_bytes: 200 }, + eth1: { rx_bytes: 11, tx_bytes: 22 } + } + }), + state: runningState() + } + const snap = buildSnapshot(raw, undefined, ALLOC, 0) + expect(snap.network).to.deep.equal({ rxBytes: 111, txBytes: 222 }) + expect(snap.blockIO).to.deep.equal({ readBytes: 1000, writeBytes: 2000 }) + expect(snap.pids.current).to.equal(7) + expect(snap.pids.limit).to.equal(512) + }) + + it('captures structured container exit info (OOM / exit code)', () => { + const raw: RawContainerSample = { + stats: cgroupV2Stats(), + state: runningState({ + Status: 'exited', + Running: false, + OOMKilled: true, + ExitCode: 137, + FinishedAt: '2026-07-28T10:05:00Z' + }) + } + const snap = buildSnapshot(raw, undefined, ALLOC, 0) + expect(snap.containerState.oomKilled).to.equal(true) + expect(snap.containerState.exitCode).to.equal(137) + expect(snap.containerState.status).to.equal('exited') + expect(snap.containerState.finishedAt).to.equal('2026-07-28T10:05:00Z') + }) + + it('tolerates cgroup v1 field drift / missing fields without throwing', () => { + // cgroup v1-ish: no online_cpus, percpu list present, memory.stats.cache instead of inactive_file + const raw: RawContainerSample = { + stats: { + cpu_stats: { + cpu_usage: { total_usage: 5e9, percpu_usage: [1, 1] }, + system_cpu_usage: 50e9, + throttling_data: {} + }, + memory_stats: { usage: 200 * MB, limit: 512 * MB, stats: { cache: 50 * MB } } + }, + state: runningState() + } + const prev = { + collectedAt: '2026-07-28T10:00:00Z', + prev: { cpuTotal: 4e9, systemCpu: 40e9, sampledAt: '2026-07-28T10:00:00Z' } + } as ContainerMetricsSnapshot + const snap = buildSnapshot(raw, prev, { cpu: 1, ramBytes: 0, diskBytes: 0 }, 0) + // Δcpu=1e9, Δsys=10e9, cpus inferred from percpu_usage length = 2 → 0.1*2*100 = 20% + expect(snap.cpu.usagePercent).to.equal(20) + expect(snap.memory.usageBytes).to.equal(150 * MB) // 200 - 50 cache + expect(snap.blockIO).to.deep.equal({ readBytes: 0, writeBytes: 0 }) + expect(snap.network).to.equal(undefined) + }) +}) + +describe('containerMetrics.buildSnapshot on a container that already exited', () => { + // An exited container's cgroup is gone: Docker reports zeros for everything. That is the + // moment the FINAL snapshot is taken, so cumulative counters must keep what the container + // actually consumed instead of resetting to 0. + const deadStats = { + cpu_stats: { cpu_usage: { total_usage: 0 }, system_cpu_usage: 0, online_cpus: 0 }, + memory_stats: {}, + blkio_stats: {}, + pids_stats: { current: 0 } + } + const prev = { + collectedAt: '2026-07-28T10:00:00Z', + cpu: { + usagePercent: 95, + allocated: 3, + usagePercentOfAllocated: 31.6, + cumulativeSeconds: 111.5, + throttledPeriods: 7, + throttledSeconds: 1.25 + }, + memory: { + usageBytes: 900 * MB, + limitBytes: 4 * GB, + usagePercent: 22, + peakUsageBytes: 1200 * MB + }, + disk: { usedBytes: 700 * MB, quotaBytes: GB, usagePercent: 68.4 }, + network: { rxBytes: 5000, txBytes: 6000 }, + blockIO: { readBytes: 7000, writeBytes: 8000 }, + pids: { current: 12, limit: 512 }, + prev: { cpuTotal: 111.5e9, systemCpu: 500e9, sampledAt: '2026-07-28T10:00:00Z' } + } as ContainerMetricsSnapshot + + const raw: RawContainerSample = { + stats: deadStats, + state: { Status: 'exited', Running: false, ExitCode: 0, OOMKilled: false } + } + + it('keeps cumulative cpu seconds, throttling, network and block IO', () => { + const snap = buildSnapshot(raw, prev, { cpu: 3, ramBytes: 4 * GB, diskBytes: GB }, 0) + expect(snap.cpu.cumulativeSeconds).to.equal(111.5) + expect(snap.cpu.throttledPeriods).to.equal(7) + expect(snap.cpu.throttledSeconds).to.equal(1.25) + expect(snap.network).to.deep.equal({ rxBytes: 5000, txBytes: 6000 }) + expect(snap.blockIO).to.deep.equal({ readBytes: 7000, writeBytes: 8000 }) + }) + + it('keeps the memory peak and the last known disk figure', () => { + // diskUsedBytes 0 = "unmeasurable" (the container is gone, `du` cannot run) + const snap = buildSnapshot(raw, prev, { cpu: 3, ramBytes: 4 * GB, diskBytes: GB }, 0) + expect(snap.memory.peakUsageBytes).to.equal(1200 * MB) + expect(snap.disk.usedBytes).to.equal(700 * MB) + expect(snap.disk.usagePercent).to.equal(68.36) + }) + + it('still reports the live gauges as 0 and records the exit info', () => { + const snap = buildSnapshot(raw, prev, { cpu: 3, ramBytes: 4 * GB, diskBytes: GB }, 0) + // nothing is running any more: instantaneous values are genuinely zero + expect(snap.cpu.usagePercent).to.equal(0) + expect(snap.memory.usageBytes).to.equal(0) + expect(snap.pids.current).to.equal(0) + expect(snap.containerState.status).to.equal('exited') + expect(snap.containerState.exitCode).to.equal(0) + }) + + it('takes a fresh measurement over the previous one when it is available', () => { + const snap = buildSnapshot( + { stats: cgroupV2Stats(), state: runningState() }, + prev, + { cpu: 3, ramBytes: 4 * GB, diskBytes: GB }, + 900 * MB + ) + expect(snap.disk.usedBytes).to.equal(900 * MB) + // cumulative CPU grew past the previous sample, so the new value wins + expect(snap.cpu.cumulativeSeconds).to.equal(111.5) + expect(snap.blockIO.readBytes).to.equal(7000) + }) +}) + +describe('containerMetrics debug formatting (formatBytes / describeSnapshot)', () => { + it('formats bytes with binary units', () => { + expect(formatBytes(0)).to.equal('0 B') + expect(formatBytes(512)).to.equal('512 B') + expect(formatBytes(1024)).to.equal('1.0 KiB') + expect(formatBytes(1536)).to.equal('1.5 KiB') + expect(formatBytes(4 * 1024 * MB)).to.equal('4.0 GiB') + // non-numbers must not produce NaN in a log line + expect(formatBytes(undefined as any)).to.equal('0 B') + }) + + it('renders a full snapshot as one greppable line', () => { + const snapshot = { + collectedAt: '2026-07-28T10:00:00Z', + containerState: { + status: 'running', + startedAt: '2026-07-28T09:00:00Z', + oomKilled: false, + restartCount: 0, + health: 'healthy' + }, + cpu: { + usagePercent: 42.5, + allocated: 2, + usagePercentOfAllocated: 21.25, + cumulativeSeconds: 12.5, + throttledPeriods: 3, + throttledSeconds: 0.25 + }, + memory: { + usageBytes: 150 * MB, + limitBytes: 512 * MB, + usagePercent: 29.3, + peakUsageBytes: 200 * MB + }, + disk: { usedBytes: 100 * MB, quotaBytes: 1024 * MB, usagePercent: 9.77 }, + network: { rxBytes: 2048, txBytes: 1024 }, + blockIO: { readBytes: 0, writeBytes: 4096 }, + pids: { current: 7, limit: 512 }, + gpu: [ + { + resourceId: 'gpu0', + vendor: 'nvidia' as const, + utilizationPercent: 88, + memoryUsedBytes: 1024 * MB, + memoryTotalBytes: 3072 * MB + } + ] + } as ContainerMetricsSnapshot + + const line = describeSnapshot(snapshot) + expect(line).to.not.contain('\n') + expect(line).to.contain('cpu 42.5% (21.25% of 2 core(s)') + expect(line).to.contain('throttled 3 periods/0.25s') + expect(line).to.contain('mem 150.0 MiB/512.0 MiB (29.3%, peak 200.0 MiB)') + expect(line).to.contain('disk 100.0 MiB/1.0 GiB (9.77%)') + expect(line).to.contain('pids 7/512') + expect(line).to.contain('net rx 2.0 KiB tx 1.0 KiB') + expect(line).to.contain('blkio r 0 B w 4.0 KiB') + expect(line).to.contain('state running') + expect(line).to.contain('health=healthy') + expect(line).to.contain('gpu gpu0=88%/1.0 GiB') + }) + + it('never throws on a minimal snapshot (a debug line must not break collection)', () => { + const minimal = { + collectedAt: '2026-07-28T10:00:00Z', + containerState: { + status: 'exited', + oomKilled: true, + exitCode: 137, + restartCount: 0 + }, + cpu: { + usagePercent: 0, + allocated: 0, + usagePercentOfAllocated: 0, + cumulativeSeconds: 0, + throttledPeriods: 0, + throttledSeconds: 0 + }, + memory: { usageBytes: 0, limitBytes: 0, usagePercent: 0, peakUsageBytes: 0 }, + disk: { usedBytes: 0 }, + blockIO: { readBytes: 0, writeBytes: 0 }, + pids: { current: 0, limit: 512 } + } as ContainerMetricsSnapshot + + const line = describeSnapshot(minimal) + // no network → "n/a" rather than a crash or a fake 0; no quota → no "/quota" part + expect(line).to.contain('net rx n/a tx n/a') + expect(line).to.contain('disk 0 B,') + expect(line).to.contain('state exited OOMKilled exit=137') + expect(line).to.not.contain('gpu ') + }) +}) + +describe('containerMetrics config helpers', () => { + const original = ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value + afterEach(() => { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = original + }) + + it('defaults the interval to 10s', () => { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = undefined + expect(getMetricsIntervalSeconds()).to.equal(10) + expect(isMetricsCollectionEnabled()).to.equal(true) + }) + + it('treats 0 as disabled', () => { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = '0' + expect(getMetricsIntervalSeconds()).to.equal(0) + expect(isMetricsCollectionEnabled()).to.equal(false) + }) + + it('falls back to the default on invalid input', () => { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = 'abc' + expect(getMetricsIntervalSeconds()).to.equal(10) + }) + + it('respects a custom cadence', () => { + ENVIRONMENT_VARIABLES.C2D_METRICS_INTERVAL_SECONDS.value = '30' + expect(getMetricsIntervalSeconds()).to.equal(30) + }) +}) + +describe('containerMetrics.isSnapshotStale', () => { + const now = new Date('2026-07-28T10:00:30Z').getTime() + it('is stale when there is no previous snapshot', () => { + expect(isSnapshotStale(undefined, 10, now)).to.equal(true) + }) + it('is stale when older than the interval', () => { + const prev = { collectedAt: '2026-07-28T10:00:15Z' } as ContainerMetricsSnapshot + expect(isSnapshotStale(prev, 10, now)).to.equal(true) // 15s old ≥ 10s + }) + it('is fresh when within the interval', () => { + const prev = { collectedAt: '2026-07-28T10:00:25Z' } as ContainerMetricsSnapshot + expect(isSnapshotStale(prev, 10, now)).to.equal(false) // 5s old < 10s + }) +}) + +describe('gpu parseMemoryTotalToBytes', () => { + it('parses MiB / GiB / GB strings', () => { + expect(parseMemoryTotalToBytes('3072 MiB')).to.equal(3072 * 1024 * 1024) + expect(parseMemoryTotalToBytes('16 GiB')).to.equal(16 * 1024 ** 3) + expect(parseMemoryTotalToBytes('8GB')).to.equal(8e9) + }) + it('returns undefined for absent/garbage input', () => { + expect(parseMemoryTotalToBytes(undefined)).to.equal(undefined) + expect(parseMemoryTotalToBytes('lots')).to.equal(undefined) + }) +}) + +describe('gpu NVML multi-GPU resolution + sampling', () => { + const gpuResource = (id: string, uuid: string): ComputeResource => ({ + id, + type: 'gpu', + kind: 'discrete', + platform: 'nvidia', + memoryTotal: '3072 MiB', + total: 1, + min: 0, + max: 1, + init: { + deviceRequests: { Driver: 'nvidia', DeviceIDs: [uuid], Capabilities: [['gpu']] } + } + }) + + it('resolves each GPU resource to its own handle (pinned by NVML UUID)', () => { + const collector = new NvmlGpuCollector() + const h0 = collector.resolve(gpuResource('gpu0', 'GPU-aaaa')) + const h1 = collector.resolve(gpuResource('gpu1', 'GPU-bbbb')) + expect(h0?.resourceId).to.equal('gpu0') + expect(h0?.uuid).to.equal('GPU-aaaa') + expect(h0?.memoryTotalBytes).to.equal(3072 * 1024 * 1024) + expect(h1?.uuid).to.equal('GPU-bbbb') + }) + + it('samples every held device — one entry per GPU (multi-GPU job)', async () => { + const collector: any = new NvmlGpuCollector() + // Inject a fake NVML binding layer (no real libnvidia-ml on CI): each getter fills its + // out-param and returns NVML_SUCCESS (0). Utilization keyed off the UUID so the two + // devices produce distinct numbers. + const utilByUuid: Record = { 'GPU-aaaa': 11, 'GPU-bbbb': 77 } + let currentUuid = '' + collector.detected = true + collector.initialized = true + collector.bindings = { + getHandleByUUID: (uuid: string, out: any[]) => { + currentUuid = uuid + out[0] = { uuid } + return 0 + }, + getUtilizationRates: (_dev: any, util: any) => { + util.gpu = utilByUuid[currentUuid] + util.memory = 0 + return 0 + }, + getMemoryInfo: (_dev: any, mem: any) => { + mem.used = 1024 * MB + mem.total = 3072 * MB + return 0 + }, + getTemperature: (_dev: any, _s: number, out: any[]) => { + out[0] = 55 + return 0 + }, + getPowerUsage: (_dev: any, out: any[]) => { + out[0] = 90000 // mW + return 0 + } + } + const handles = [ + collector.resolve(gpuResource('gpu0', 'GPU-aaaa')), + collector.resolve(gpuResource('gpu1', 'GPU-bbbb')) + ] + const metrics = await collector.sample(handles) + expect(metrics).to.have.length(2) + expect(metrics[0].resourceId).to.equal('gpu0') + expect(metrics[0].utilizationPercent).to.equal(11) + expect(metrics[0].memoryUsedBytes).to.equal(1024 * MB) + expect(metrics[0].powerWatts).to.equal(90) + expect(metrics[1].resourceId).to.equal('gpu1') + expect(metrics[1].utilizationPercent).to.equal(77) + }) +}) + +describe('runtimeMetrics is stripped from every public shape (default)', () => { + // A snapshot carrying the internal `prev` accumulator, which must never surface publicly. + const snapshot = { + collectedAt: '2026-07-28T10:00:00Z', + containerState: { status: 'running', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 40, + allocated: 2, + usagePercentOfAllocated: 20, + cumulativeSeconds: 1, + throttledPeriods: 0, + throttledSeconds: 0 + }, + prev: { cpuTotal: 1e9, systemCpu: 10e9, sampledAt: '2026-07-28T10:00:00Z' } + } as ContainerMetricsSnapshot + + it('omitDBComputeFieldsFromComputeJob drops runtimeMetrics by default (status + escrow proof)', () => { + const dbJob = { + jobId: 'job-1', + owner: '0xabc', + runtimeMetrics: snapshot, + clusterHash: 'h', + resources: [] + } as unknown as DBComputeJob + const pub = omitDBComputeFieldsFromComputeJob(dbJob) + expect('runtimeMetrics' in (pub as any)).to.equal(false) + // and it must not appear in the serialized escrow-proof form either + expect(JSON.stringify(pub)).to.not.contain('runtimeMetrics') + }) + + it('toPublicServiceJob and toListedServiceJob drop runtimeMetrics by default', () => { + const svc = { + serviceId: 's-1', + owner: '0xabc', + userData: 'enc', + runtimeMetrics: snapshot + } as unknown as ServiceJob + const pub = toPublicServiceJob(svc) + const listed = toListedServiceJob(svc) + expect('runtimeMetrics' in (pub as any)).to.equal(false) + expect('runtimeMetrics' in (listed as any)).to.equal(false) + }) +}) + +describe('runtimeMetrics opt-in exposure (owner status path)', () => { + const snapshot = { + collectedAt: '2026-07-28T10:00:00Z', + containerState: { status: 'running', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 40, + allocated: 2, + usagePercentOfAllocated: 20, + cumulativeSeconds: 1, + throttledPeriods: 0, + throttledSeconds: 0 + }, + prev: { cpuTotal: 1e9, systemCpu: 10e9, sampledAt: '2026-07-28T10:00:00Z' } + } as ContainerMetricsSnapshot + + it('sanitizePublicMetrics drops the internal prev accumulator', () => { + const pub = sanitizePublicMetrics(snapshot) + expect(pub).to.not.equal(undefined) + expect('prev' in (pub as any)).to.equal(false) + expect(pub!.cpu.usagePercent).to.equal(40) // real metrics retained + }) + + it('omitDBComputeFieldsFromComputeJob keeps sanitized metrics with { includeMetrics: true }', () => { + const dbJob = { + jobId: 'job-1', + owner: '0xabc', + runtimeMetrics: snapshot, + clusterHash: 'h', + resources: [] + } as unknown as DBComputeJob + const pub = omitDBComputeFieldsFromComputeJob(dbJob, { includeMetrics: true }) + expect((pub as any).runtimeMetrics).to.not.equal(undefined) + // still strips other internal fields, and the exposed snapshot drops `prev` + expect('clusterHash' in (pub as any)).to.equal(false) + expect('prev' in (pub as any).runtimeMetrics).to.equal(false) + expect((pub as any).runtimeMetrics.cpu.usagePercent).to.equal(40) + }) + + it('the escrow-proof (default) shape stays metrics-free regardless', () => { + const dbJob = { + jobId: 'job-1', + owner: '0xabc', + runtimeMetrics: snapshot, + resources: [] + } as unknown as DBComputeJob + // Default call = the proof shape at compute_engine_docker.ts + const proof = JSON.stringify(omitDBComputeFieldsFromComputeJob(dbJob)) + expect(proof).to.not.contain('runtimeMetrics') + expect(proof).to.not.contain('"prev"') + }) + + it('toPublicServiceJob keeps sanitized metrics with { includeMetrics: true } but still strips userData', () => { + const svc = { + serviceId: 's-1', + owner: '0xabc', + userData: 'enc', + runtimeMetrics: snapshot + } as unknown as ServiceJob + const pub = toPublicServiceJob(svc, { includeMetrics: true }) as any + expect(pub.runtimeMetrics).to.not.equal(undefined) + expect('prev' in pub.runtimeMetrics).to.equal(false) + expect('userData' in pub).to.equal(false) + // the node-wide listing shape never exposes metrics, even now + expect('runtimeMetrics' in (toListedServiceJob(svc) as any)).to.equal(false) + }) +}) + +describe('resolveMetricsRequestMode (COMPUTE_GET_STATUS default)', () => { + const consumerAddress = '0x7C8226E267Cd509bCBE12B4e69fbE07052422Dbd' + const base = { command: PROTOCOL_COMMANDS.COMPUTE_GET_STATUS, jobId: 'job-1' } + + it('defaults to best-effort when the caller carries signature credentials', () => { + const task = { ...base, consumerAddress, nonce: '1', signature: '0xsig' } as any + expect(resolveMetricsRequestMode(task)).to.equal('best-effort') + }) + + it('defaults to best-effort when the caller carries an auth token', () => { + const task = { ...base, consumerAddress, authorization: 'Bearer tok' } as any + expect(resolveMetricsRequestMode(task)).to.equal('best-effort') + }) + + it('stays off for the unauthenticated status call (no credentials to check)', () => { + expect(resolveMetricsRequestMode({ ...base } as any)).to.equal('off') + // consumerAddress alone proves nothing + expect(resolveMetricsRequestMode({ ...base, consumerAddress } as any)).to.equal('off') + // a half-supplied signature pair is not credentials either + expect( + resolveMetricsRequestMode({ ...base, consumerAddress, nonce: '1' } as any) + ).to.equal('off') + expect( + resolveMetricsRequestMode({ ...base, consumerAddress, signature: '0xsig' } as any) + ).to.equal('off') + }) + + it('is required when explicitly requested, even with nothing to authenticate with', () => { + expect(resolveMetricsRequestMode({ ...base, includeMetrics: true } as any)).to.equal( + 'required' + ) + expect( + resolveMetricsRequestMode({ + ...base, + consumerAddress, + nonce: '1', + signature: '0xsig', + includeMetrics: true + } as any) + ).to.equal('required') + }) + + it('is off when explicitly opted out, whatever credentials are present', () => { + expect( + resolveMetricsRequestMode({ + ...base, + consumerAddress, + nonce: '1', + signature: '0xsig', + includeMetrics: false + } as any) + ).to.equal('off') + }) +}) diff --git a/src/test/unit/evmAddress.test.ts b/src/test/unit/evmAddress.test.ts new file mode 100644 index 000000000..ee4599170 --- /dev/null +++ b/src/test/unit/evmAddress.test.ts @@ -0,0 +1,140 @@ +import { expect } from 'chai' +import { describe, it } from 'mocha' +import { + addressCasingVariants, + includesAddress, + normalizeAddress, + normalizeAddresses, + normalizeCommandAddresses, + sameAddress +} from '../../utils/evmAddress.js' + +const CHECKSUMMED = '0x7C8226E267Cd509bCBE12B4e69fbE07052422Dbd' +const LOWERCASE = CHECKSUMMED.toLowerCase() +const UPPERCASE = '0x' + CHECKSUMMED.slice(2).toUpperCase() + +describe('normalizeAddress', () => { + it('checksums a lowercase address', () => { + expect(normalizeAddress(LOWERCASE)).to.equal(CHECKSUMMED) + }) + + it('leaves an already checksummed address untouched', () => { + expect(normalizeAddress(CHECKSUMMED)).to.equal(CHECKSUMMED) + }) + + it('checksums an all-uppercase address', () => { + expect(normalizeAddress(UPPERCASE)).to.equal(CHECKSUMMED) + }) + + it('returns non-addresses unchanged so validators can still reject them', () => { + // a mixed-case string with a BAD checksum is not a valid address: it must reach the + // command validator as-is instead of throwing here + const badChecksum = '0x7c8226E267Cd509bCBE12B4e69fbE07052422DBD' + expect(normalizeAddress(badChecksum)).to.equal(badChecksum) + expect(normalizeAddress('not-an-address')).to.equal('not-an-address') + expect(normalizeAddress('')).to.equal('') + expect(normalizeAddress(undefined as any)).to.equal(undefined) + expect(normalizeAddress(null as any)).to.equal(null) + }) +}) + +describe('normalizeAddresses', () => { + it('normalizes each entry and passes non-arrays through', () => { + expect(normalizeAddresses([LOWERCASE, 'nope'])).to.deep.equal([CHECKSUMMED, 'nope']) + expect(normalizeAddresses(undefined as any)).to.equal(undefined) + }) +}) + +describe('sameAddress / includesAddress', () => { + it('compares addresses case-insensitively', () => { + expect(sameAddress(LOWERCASE, CHECKSUMMED)).to.equal(true) + expect(sameAddress(UPPERCASE, LOWERCASE)).to.equal(true) + }) + + it('is false for different addresses and for missing operands', () => { + expect(sameAddress(CHECKSUMMED, '0x' + '1'.repeat(40))).to.equal(false) + expect(sameAddress(undefined, CHECKSUMMED)).to.equal(false) + expect(sameAddress(CHECKSUMMED, undefined)).to.equal(false) + expect(sameAddress('', '')).to.equal(false) + }) + + it('matches list membership case-insensitively', () => { + expect(includesAddress([LOWERCASE], CHECKSUMMED)).to.equal(true) + expect(includesAddress([CHECKSUMMED], LOWERCASE)).to.equal(true) + expect(includesAddress([], CHECKSUMMED)).to.equal(false) + expect(includesAddress(undefined, CHECKSUMMED)).to.equal(false) + expect(includesAddress([CHECKSUMMED], undefined)).to.equal(false) + }) +}) + +describe('addressCasingVariants', () => { + it('offers the canonical and lowercase forms for a lowercase input, deduped', () => { + const variants = addressCasingVariants(LOWERCASE) + expect(variants).to.deep.equal([LOWERCASE, CHECKSUMMED]) + }) + + it('offers both forms for a checksummed input, canonical first', () => { + const variants = addressCasingVariants(CHECKSUMMED) + expect(variants).to.deep.equal([CHECKSUMMED, LOWERCASE]) + }) + + it('never returns duplicates', () => { + for (const input of [LOWERCASE, CHECKSUMMED, UPPERCASE]) { + const variants = addressCasingVariants(input) + expect(variants.length).to.equal(new Set(variants).size) + } + }) + + it('passes a non-string through as a single candidate', () => { + expect(addressCasingVariants(undefined as any)).to.deep.equal([undefined]) + }) +}) + +describe('normalizeCommandAddresses', () => { + it('canonicalizes every known address field in place', () => { + const task: any = { + command: 'computeGetStatus', + consumerAddress: LOWERCASE, + owner: LOWERCASE, + address: LOWERCASE, + decrypterAddress: LOWERCASE, + dataNftAddress: LOWERCASE, + publisherAddress: LOWERCASE, + consumerAddrs: [LOWERCASE, UPPERCASE], + additionalViewers: [LOWERCASE] + } + const returned = normalizeCommandAddresses(task) + expect(returned).to.equal(task) // mutated in place + expect(task.consumerAddress).to.equal(CHECKSUMMED) + expect(task.owner).to.equal(CHECKSUMMED) + expect(task.address).to.equal(CHECKSUMMED) + expect(task.decrypterAddress).to.equal(CHECKSUMMED) + expect(task.dataNftAddress).to.equal(CHECKSUMMED) + expect(task.publisherAddress).to.equal(CHECKSUMMED) + expect(task.consumerAddrs).to.deep.equal([CHECKSUMMED, CHECKSUMMED]) + expect(task.additionalViewers).to.deep.equal([CHECKSUMMED]) + }) + + it('leaves unrelated fields, absent fields and invalid values alone', () => { + const task: any = { + command: 'computeGetStatus', + jobId: '0xNOTanADDRESS', + // a JWT-ish token must not be touched even though it is a string field + authorization: 'Bearer abc.def.ghi', + consumerAddress: 'not-an-address', + signature: '0xdeadbeef' + } + normalizeCommandAddresses(task) + expect(task.jobId).to.equal('0xNOTanADDRESS') + expect(task.authorization).to.equal('Bearer abc.def.ghi') + expect(task.consumerAddress).to.equal('not-an-address') + expect(task.signature).to.equal('0xdeadbeef') + expect('owner' in task).to.equal(false) + }) + + it('tolerates null/undefined/non-object input', () => { + expect(normalizeCommandAddresses(null)).to.equal(null) + expect(normalizeCommandAddresses(undefined)).to.equal(undefined) + expect(normalizeCommandAddresses('x' as any)).to.equal('x') + }) +}) diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index 7a797a9d3..d66dfd15a 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -13,7 +13,9 @@ import { ServiceExtendHandler } from '../../../components/core/service/extendSer import { ServiceRestartHandler } from '../../../components/core/service/restartService.js' import { ServiceGetStreamableLogsHandler } from '../../../components/core/service/getStreamableLogs.js' -const OWNER = '0x0000000000000000000000000000000000000abc' +// Checksummed (EIP-55): commands are canonicalized on ingress, so this is the form handlers +// see and forward, whatever casing the caller sent (see the lowercase-address test below). +const OWNER = '0x0000000000000000000000000000000000000aBc' function makeJob(overrides: Partial = {}): ServiceJob { return { @@ -171,7 +173,9 @@ function buildFakes(opts: FakeOpts = {}) { decrypt: (d: Uint8Array) => Promise.resolve(Buffer.from(d)) }), getAuth: () => ({ - validateAuthenticationOrToken: () => Promise.resolve({ valid: true }) + // mirrors the real signature path: the address that verified IS the claimed one + validateAuthenticationOrToken: ({ address }: any) => + Promise.resolve({ valid: true, address }) }) } @@ -224,6 +228,26 @@ describe('Service handlers', () => { expect(res.status.httpStatus).to.equal(401) }) + it('401 when an auth token belongs to a different address than the one claimed', async () => { + // An auth token authenticates whoever it was ISSUED to. Claiming someone else's + // consumerAddress while presenting your own token must not read their services. + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + node.getAuth = () => ({ + validateAuthenticationOrToken: () => + Promise.resolve({ + valid: true, + address: '0x000000000000000000000000000000000000dEaD' + }) + }) + const res = await new ServiceGetStatusHandler(node).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress: OWNER, + authorization: 'Bearer someone-elses-token', + serviceId: 'svc-1' + } as any) + expect(res.status.httpStatus).to.equal(401) + }) + it('returns jobs by serviceId with userData stripped (authenticated)', async () => { const { node } = buildFakes({ serviceJobInDb: makeJob() }) const res = await new ServiceGetStatusHandler(node).handle({ @@ -271,6 +295,19 @@ describe('Service handlers', () => { expect(jobs[0].status).to.equal(ServiceStatusNumber.Stopped) expect(jobs[0]).to.not.have.property('userData') }) + + it('accepts a non-checksummed consumerAddress and queries with the canonical form', async () => { + // An address is an identity KEY here (the `owner` filter on the job lookup), so a + // lowercase address must not read as "service not found". + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceStopHandler(node).handle({ + ...baseTask, + consumerAddress: OWNER.toLowerCase() + } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.stopService.calledOnce).to.equal(true) + expect(engine.db.getServiceJob.firstCall.args[1]).to.equal(OWNER) + }) }) describe('ServiceRestartHandler', () => { diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index 2eaf55eab..ad6992897 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -475,4 +475,86 @@ describe('Service Jobs Database', () => { expect(threw).to.equal(true) }) }) + + describe('updateServiceJobMetrics (guarded, metrics-only write)', () => { + const snapshot: any = { + collectedAt: '2026-07-29T12:00:00Z', + containerState: { status: 'running', oomKilled: false, restartCount: 0 }, + cpu: { + usagePercent: 40, + allocated: 2, + usagePercentOfAllocated: 20, + cumulativeSeconds: 1, + throttledPeriods: 0, + throttledSeconds: 0 + } + } + + it('persists runtimeMetrics and preserves lifecycle fields when the row still matches', async () => { + const expiresAt = Date.now() + 7200_000 + const job = makeServiceJob({ status: ServiceStatusNumber.Running, expiresAt }) + await db.newServiceJob(job) + + const wrote = await db.updateServiceJobMetrics( + job.serviceId, + { + owner: job.owner, + clusterHash: job.clusterHash, + status: ServiceStatusNumber.Running, + containerId: job.containerId + }, + snapshot + ) + expect(wrote).to.equal(true) + + const [fresh] = await db.getServiceJob(job.serviceId, job.owner) + expect(fresh.runtimeMetrics?.collectedAt).to.equal('2026-07-29T12:00:00Z') + // lifecycle fields untouched + expect(fresh.status).to.equal(ServiceStatusNumber.Running) + expect(fresh.expiresAt).to.equal(expiresAt) + expect(fresh.containerId).to.equal(job.containerId) + }) + + it('is a no-op when status changed (guards against clobbering a lifecycle transition)', async () => { + const job = makeServiceJob({ status: ServiceStatusNumber.Running }) + await db.newServiceJob(job) + + const wrote = await db.updateServiceJobMetrics( + job.serviceId, + { + owner: job.owner, + clusterHash: job.clusterHash, + status: ServiceStatusNumber.Stopped, // does not match the persisted Running + containerId: job.containerId + }, + snapshot + ) + expect(wrote).to.equal(false) + + const [fresh] = await db.getServiceJob(job.serviceId, job.owner) + expect(fresh.runtimeMetrics).to.equal(undefined) + }) + + it('is a no-op when the container was replaced (restart)', async () => { + const job = makeServiceJob({ + status: ServiceStatusNumber.Running, + containerId: 'container-old' + }) + await db.newServiceJob(job) + + const wrote = await db.updateServiceJobMetrics( + job.serviceId, + { + owner: job.owner, + clusterHash: job.clusterHash, + status: ServiceStatusNumber.Running, + containerId: 'container-new' // differs from persisted + }, + snapshot + ) + expect(wrote).to.equal(false) + const [fresh] = await db.getServiceJob(job.serviceId, job.owner) + expect(fresh.runtimeMetrics).to.equal(undefined) + }) + }) }) diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 507b1b1bb..b6ac4c26c 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -496,6 +496,16 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DOCKER_COMPUTE_ENVIRONMENTS, required: false }, + C2D_METRICS_INTERVAL_SECONDS: { + name: 'C2D_METRICS_INTERVAL_SECONDS', + value: process.env.C2D_METRICS_INTERVAL_SECONDS, + required: false + }, + GPU_METRICS: { + name: 'GPU_METRICS', + value: process.env.GPU_METRICS, + required: false + }, SERVICE_TEMPLATES_PATH: { name: 'SERVICE_TEMPLATES_PATH', value: process.env.SERVICE_TEMPLATES_PATH, diff --git a/src/utils/evmAddress.ts b/src/utils/evmAddress.ts new file mode 100644 index 000000000..0e28dc72d --- /dev/null +++ b/src/utils/evmAddress.ts @@ -0,0 +1,96 @@ +import { ethers } from 'ethers' + +// EVM addresses reach the node in whatever casing the client happened to use: checksummed +// (EIP-55) straight from a wallet, all-lowercase out of a database, or hand-typed in a curl. +// The node then uses them as identity KEYS — the `owner` filter on job/service queries, the +// nonce row id, ownership comparisons. A differently-cased address does not fail loudly +// there, it silently MISSES: the job "does not exist", or an owner-only field is quietly +// dropped. So every address a user supplies is canonicalized on ingress +// (`normalizeCommandAddresses`, called from the handler base classes) and every ownership +// comparison stays case-insensitive on top of that. +// +// (This module is about the addresses *callers send us*. `utils/address.ts` is a different +// thing entirely: the deployed Ocean contract artifact addresses per chain.) + +/** + * Canonical EIP-55 (checksummed) form of an address. + * Anything that is not a valid address is returned untouched, so the per-command validators + * still produce their own "not a valid web3 address" response instead of this throwing. + */ +export function normalizeAddress(value: string): string { + if (typeof value !== 'string' || !ethers.isAddress(value)) return value + return ethers.getAddress(value) +} + +/** normalizeAddress over a list, leaving non-array input untouched. */ +export function normalizeAddresses(values: string[]): string[] { + if (!Array.isArray(values)) return values + return values.map((value) => normalizeAddress(value)) +} + +/** + * Case-insensitive address equality — use it for EVERY ownership/authorization comparison. + * Ingress normalization fixes what the caller sends; the other side of the comparison can + * still be a legacy DB row written before normalization existed. + */ +export function sameAddress(a?: string, b?: string): boolean { + if (!a || !b) return false + return a.toLowerCase() === b.toLowerCase() +} + +/** Case-insensitive `list.includes(address)`. Null-safe on both sides. */ +export function includesAddress(list?: string[], address?: string): boolean { + if (!list || !address) return false + return list.some((entry) => sameAddress(entry, address)) +} + +/** + * Every plausible casing a client could have embedded in a SIGNED message for this address, + * most-likely first and deduped. + * + * Signature verification MUST try all of them: the signed message is built client-side by + * concatenating the address as a string, so verifying only the canonical form would reject + * signatures produced over the lowercase form — a compatibility break introduced by the + * ingress normalization itself. + */ +export function addressCasingVariants(value: string): string[] { + if (typeof value !== 'string') return [value] + return [...new Set([value, normalizeAddress(value), value.toLowerCase()])] +} + +// Command fields carrying a single EVM address supplied by the caller. +// Deliberately NOT included: the escrow query fields (`payer`/`payee`/`token`), whose handler +// lowercases them to match how escrow events are stored (see escrowHandler). +const ADDRESS_FIELDS = [ + 'consumerAddress', + 'address', + 'owner', + 'decrypterAddress', + 'dataNftAddress', + 'publisherAddress' +] + +// Command fields carrying a list of caller-supplied EVM addresses. +const ADDRESS_LIST_FIELDS = ['consumerAddrs', 'additionalViewers'] + +/** + * Canonicalizes every caller-supplied address on a command, in place, before anything reads + * it (validators, DB lookups, ownership checks). Called once from the handler base classes so + * it covers all three entry points — REST routes, HTTP /directCommand and P2P — for every + * command, present and future. + */ +export function normalizeCommandAddresses(task: T): T { + if (!task || typeof task !== 'object') return task + const record = task as Record + for (const field of ADDRESS_FIELDS) { + if (typeof record[field] === 'string') { + record[field] = normalizeAddress(record[field]) + } + } + for (const field of ADDRESS_LIST_FIELDS) { + if (Array.isArray(record[field])) { + record[field] = normalizeAddresses(record[field]) + } + } + return task +}